CHEF-3781: Merge the latest knife-essentials (1.3), including:

- Huge performance boost (parallel download/upload/diff)
- Support for Hosted data (acls, groups, containers) for full backup/restore of organizations
- knife deps: dependency tree viewing for objects
- knife xargs, knife edit
- knife integration testing
- Support for multiple cookbook_paths, chef_repo_path config option
- Windows backslash support
- Many edge case fixes, better output and some new command line options

64 files changed, 3857 insertions, 712 deletions

diff --git a/lib/chef/chef_fs.rb b/lib/chef/chef_fs.rb
index 14ab8c0a6e..bc445e53ad 100644
--- a/lib/chef/chef_fs.rb
+++ b/lib/chef/chef_fs.rb
@@ -1,11 +1,9 @@
-require 'chef/chef_fs/file_system/chef_server_root_dir'
-require 'chef/config'
-require 'chef/rest'
+require 'chef/platform'
 
 class Chef
   module ChefFS
     def self.windows?
-      false
+      Chef::Platform.windows?
     end
   end
 end
diff --git a/lib/chef/chef_fs/chef_fs_data_store.rb b/lib/chef/chef_fs/chef_fs_data_store.rb
new file mode 100644
index 0000000000..bd8226b92f
--- /dev/null
+++ b/lib/chef/chef_fs/chef_fs_data_store.rb
@@ -0,0 +1,375 @@
+#
+# Author:: John Keiser (<jkeiser@opscode.com>)
+# Copyright:: Copyright (c) 2012 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef_zero/data_store/memory_store'
+require 'chef_zero/data_store/data_already_exists_error'
+require 'chef_zero/data_store/data_not_found_error'
+require 'chef/chef_fs/file_pattern'
+require 'chef/chef_fs/file_system'
+require 'chef/chef_fs/file_system/not_found_error'
+require 'chef/chef_fs/file_system/memory_root'
+
+class Chef
+  module ChefFS
+    class ChefFSDataStore
+      def initialize(chef_fs)
+        @chef_fs = chef_fs
+        @memory_store = ChefZero::DataStore::MemoryStore.new
+      end
+
+      def publish_description
+        "Reading and writing data to #{chef_fs.fs_description}"
+      end
+
+      def chef_fs
+        @chef_fs.call
+      end
+
+      MEMORY_PATHS = %w(sandboxes file_store)
+
+      def create_dir(path, name, *options)
+        if use_memory_store?(path)
+          @memory_store.create_dir(path, name, *options)
+        else
+          with_dir(path) do |parent|
+            parent.create_child(chef_fs_filename(path + [name]), nil)
+          end
+        end
+      end
+
+      def create(path, name, data, *options)
+        if use_memory_store?(path)
+          @memory_store.create(path, name, data, *options)
+
+        elsif path[0] == 'cookbooks' && path.length == 2
+          # Do nothing.  The entry gets created when the cookbook is created.
+
+        else
+          if !data.is_a?(String)
+            raise "set only works with strings"
+          end
+
+          with_dir(path) do |parent|
+            parent.create_child(chef_fs_filename(path + [name]), data)
+          end
+        end
+      end
+
+      def get(path, request=nil)
+        if use_memory_store?(path)
+          @memory_store.get(path)
+
+        elsif path[0] == 'file_store' && path[1] == 'repo'
+          entry = Chef::ChefFS::FileSystem.resolve_path(chef_fs, path[2..-1].join('/'))
+          begin
+            entry.read
+          rescue Chef::ChefFS::FileSystem::NotFoundError => e
+            raise ChefZero::DataStore::DataNotFoundError.new(to_zero_path(e.entry), e)
+          end
+
+        else
+          with_entry(path) do |entry|
+            if path[0] == 'cookbooks' && path.length == 3
+              # get /cookbooks/NAME/version
+              result = entry.chef_object.to_hash
+              result.each_pair do |key, value|
+                if value.is_a?(Array)
+                  value.each do |file|
+                    if file.is_a?(Hash) && file.has_key?('checksum')
+                      relative = ['file_store', 'repo', 'cookbooks']
+                      if Chef::Config.versioned_cookbooks
+                        relative << "#{path[1]}-#{path[2]}"
+                      else
+                        relative << path[1]
+                      end
+                      relative = relative + file[:path].split('/')
+                      file['url'] = ChefZero::RestBase::build_uri(request.base_uri, relative)
+                    end
+                  end
+                end
+              end
+              JSON.pretty_generate(result)
+
+            else
+              entry.read
+            end
+          end
+        end
+      end
+
+      def set(path, data, *options)
+        if use_memory_store?(path)
+          @memory_store.set(path, data, *options)
+        else
+          if !data.is_a?(String)
+            raise "set only works with strings: #{path} = #{data.inspect}"
+          end
+
+          # Write out the files!
+          if path[0] == 'cookbooks' && path.length == 3
+            write_cookbook(path, data, *options)
+          else
+            with_dir(path[0..-2]) do |parent|
+              parent.create_child(chef_fs_filename(path), data)
+            end
+          end
+        end
+      end
+
+      def delete(path)
+        if use_memory_store?(path)
+          @memory_store.delete(path)
+        else
+          with_entry(path) do |entry|
+            if path[0] == 'cookbooks' && path.length >= 3
+              entry.delete(true)
+            else
+              entry.delete
+            end
+          end
+        end
+      end
+
+      def delete_dir(path, *options)
+        if use_memory_store?(path)
+          @memory_store.delete_dir(path, *options)
+        else
+          with_entry(path) do |entry|
+            entry.delete(options.include?(:recursive))
+          end
+        end
+      end
+
+      def list(path)
+        if use_memory_store?(path)
+          @memory_store.list(path)
+
+        elsif path[0] == 'cookbooks' && path.length == 1
+          with_entry(path) do |entry|
+            begin
+              if Chef::Config.versioned_cookbooks
+                # /cookbooks/name-version -> /cookbooks/name
+                entry.children.map { |child| split_name_version(child.name)[0] }.uniq
+              else
+                entry.children.map { |child| child.name }
+              end
+            rescue Chef::ChefFS::FileSystem::NotFoundError
+              # If the cookbooks dir doesn't exist, we have no cookbooks (not 404)
+              []
+            end
+          end
+
+        elsif path[0] == 'cookbooks' && path.length == 2
+          if Chef::Config.versioned_cookbooks
+            # list /cookbooks/name = filter /cookbooks/name-version down to name
+            entry.children.map { |child| split_name_version(child.name) }.
+                           select { |name, version| name == path[1] }.
+                           map { |name, version| version }.to_a
+          else
+            # list /cookbooks/name = <single version>
+            version = get_single_cookbook_version(path)
+            [version]
+          end
+
+        else
+          with_entry(path) do |entry|
+            begin
+              entry.children.map { |c| zero_filename(c) }.sort
+            rescue Chef::ChefFS::FileSystem::NotFoundError
+              # /cookbooks, /data, etc. never return 404
+              if path_always_exists?(path)
+                []
+              else
+                raise
+              end
+            end
+          end
+        end
+      end
+
+      def exists?(path)
+        if use_memory_store?(path)
+          @memory_store.exists?(path)
+        else
+          path_always_exists?(path) || Chef::ChefFS::FileSystem.resolve_path(chef_fs, to_chef_fs_path(path)).exists?
+        end
+      end
+
+      def exists_dir?(path)
+        if use_memory_store?(path)
+          @memory_store.exists_dir?(path)
+        elsif path[0] == 'cookbooks' && path.length == 2
+          list([ path[0] ]).include?(path[1])
+        else
+          Chef::ChefFS::FileSystem.resolve_path(chef_fs, to_chef_fs_path(path)).exists?
+        end
+      end
+
+      private
+
+      def use_memory_store?(path)
+        return path[0] == 'sandboxes' || path[0] == 'file_store' && path[1] == 'checksums' || path == [ 'environments', '_default' ]
+      end
+
+      def write_cookbook(path, data, *options)
+        # Create a little Chef::ChefFS memory filesystem with the data
+        if Chef::Config.versioned_cookbooks
+          cookbook_path = "cookbooks/#{path[1]}-#{path[2]}"
+        else
+          cookbook_path = "cookbooks/#{path[1]}"
+        end
+        cookbook_fs = Chef::ChefFS::FileSystem::MemoryRoot.new('uploading')
+        cookbook = JSON.parse(data, :create_additions => false)
+        cookbook.each_pair do |key, value|
+          if value.is_a?(Array)
+            value.each do |file|
+              if file.is_a?(Hash) && file.has_key?('checksum')
+                file_data = @memory_store.get(['file_store', 'checksums', file['checksum']])
+                cookbook_fs.add_file("#{cookbook_path}/#{file['path']}", file_data)
+              end
+            end
+          end
+        end
+
+        # Use the copy/diff algorithm to copy it down so we don't destroy
+        # chefignored data.  This is terribly un-thread-safe.
+        Chef::ChefFS::FileSystem.copy_to(Chef::ChefFS::FilePattern.new("/#{cookbook_path}"), cookbook_fs, chef_fs, nil, {:purge => true})
+      end
+
+      def split_name_version(entry_name)
+        name_version = entry_name.split('-')
+        name = name_version[0..-2].join('-')
+        version = name_version[-1]
+        [name,version]
+      end
+
+      def to_chef_fs_path(path)
+        _to_chef_fs_path(path).join('/')
+      end
+
+      def chef_fs_filename(path)
+        _to_chef_fs_path(path)[-1]
+      end
+
+      def _to_chef_fs_path(path)
+        if path[0] == 'data'
+          path = path.dup
+          path[0] = 'data_bags'
+          if path.length >= 3
+            path[2] = "#{path[2]}.json"
+          end
+        elsif path[0] == 'cookbooks'
+          if path.length == 2
+            raise ChefZero::DataStore::DataNotFoundError.new(path)
+          elsif Chef::Config.versioned_cookbooks
+            if path.length >= 3
+              # cookbooks/name/version -> cookbooks/name-version
+              path = [ path[0], "#{path[1]}-#{path[2]}" ] + path[3..-1]
+            end
+          else
+            if path.length >= 3
+              # cookbooks/name/version/... -> /cookbooks/name/... iff metadata says so
+              version = get_single_cookbook_version(path)
+              if path[2] == version
+                path = path[0..1] + path[3..-1]
+              else
+                raise ChefZero::DataStore::DataNotFoundError.new(path)
+              end
+            end
+          end
+        elsif path.length == 2
+          path = path.dup
+          path[1] = "#{path[1]}.json"
+        end
+        path
+      end
+
+      def to_zero_path(entry)
+        path = entry.path.split('/')[1..-1]
+        if path[0] == 'data_bags'
+          path = path.dup
+          path[0] = 'data'
+          if path.length >= 3
+            path[2] = path[2][0..-6]
+          end
+
+        elsif path[0] == 'cookbooks'
+          if Chef::Config.versioned_cookbooks
+            # cookbooks/name-version/... -> cookbooks/name/version/...
+            if path.length >= 2
+              name, version = split_name_version(path[1])
+              path = [ path[0], name, version ] + path[2..-1]
+            end
+          else
+            if path.length >= 2
+              # cookbooks/name/... -> cookbooks/name/version/...
+              version = get_single_cookbook_version(path)
+              path = path[0..1] + [version] + path[2..-1]
+            end
+          end
+
+        elsif path.length == 2 && path[0] != 'cookbooks'
+          path = path.dup
+          path[1] = path[1][0..-6]
+        end
+        path
+      end
+
+      def zero_filename(entry)
+        to_zero_path(entry)[-1]
+      end
+
+      def path_always_exists?(path)
+        return path.length == 1 && %w(clients cookbooks data environments nodes roles users).include?(path[0])
+      end
+
+      def with_entry(path)
+        begin
+          yield Chef::ChefFS::FileSystem.resolve_path(chef_fs, to_chef_fs_path(path))
+        rescue Chef::ChefFS::FileSystem::NotFoundError => e
+          raise ChefZero::DataStore::DataNotFoundError.new(to_zero_path(e.entry), e)
+        end
+      end
+
+      def with_dir(path)
+        begin
+          yield get_dir(_to_chef_fs_path(path), true)
+        rescue Chef::ChefFS::FileSystem::NotFoundError => e
+          raise ChefZero::DataStore::DataNotFoundError.new(to_zero_path(e.entry), e)
+        end
+      end
+
+      def get_dir(path, create=false)
+        result = Chef::ChefFS::FileSystem.resolve_path(chef_fs, path.join('/'))
+        if result.exists?
+          result
+        elsif create
+          get_dir(path[0..-2], create).create_child(result.name, nil)
+        else
+          raise ChefZero::DataStore::DataNotFoundError.new(path)
+        end
+      end
+
+      def get_single_cookbook_version(path)
+        dir = Chef::ChefFS::FileSystem.resolve_path(chef_fs, path[0..1].join('/'))
+        metadata = ChefZero::CookbookData.metadata_from(dir, path[1], nil, [])
+        metadata[:version] || '0.0.0'
+      end
+    end
+  end
+end
diff --git a/lib/chef/chef_fs/command_line.rb b/lib/chef/chef_fs/command_line.rb
index a8362b962b..dd5e62b755 100644
--- a/lib/chef/chef_fs/command_line.rb
+++ b/lib/chef/chef_fs/command_line.rb
@@ -17,89 +17,171 @@
 #
 
 require 'chef/chef_fs/file_system'
+require 'chef/chef_fs/file_system/operation_failed_error'
+require 'chef/chef_fs/file_system/operation_not_allowed_error'
 
 class Chef
   module ChefFS
     module CommandLine
-      def self.diff(pattern, a_root, b_root, recurse_depth, output_mode)
-        found_result = false
-        Chef::ChefFS::FileSystem.list_pairs(pattern, a_root, b_root) do |a, b|
-          existed = diff_entries(a, b, recurse_depth, output_mode) do |diff|
-            yield diff
-          end
-          found_result = true if existed
-        end
-        if !found_result && pattern.exact_path
-          yield "#{pattern}: No such file or directory on remote or local"
+
+      def self.diff_print(pattern, a_root, b_root, recurse_depth, output_mode, format_path = nil, diff_filter = nil, ui = nil)
+        if format_path.nil?
+          format_path = proc { |entry| entry.path_for_printing }
         end
-      end
 
-      # Diff two known entries (could be files or dirs)
-      def self.diff_entries(old_entry, new_entry, recurse_depth, output_mode)
-        # If both are directories
-        if old_entry.dir?
-          if new_entry.dir?
-            if recurse_depth == 0
-              if output_mode != :name_only && output_mode != :name_status
-                yield "Common subdirectories: #{old_entry.path}\n"
-              end
+        get_content = (output_mode != :name_only && output_mode != :name_status)
+        found_match = false
+        diff(pattern, a_root, b_root, recurse_depth, get_content).each do |type, old_entry, new_entry, old_value, new_value, error|
+          found_match = true unless type == :both_nonexistent
+          old_path = format_path.call(old_entry)
+          new_path = format_path.call(new_entry)
+
+          case type
+          when :common_subdirectories
+            if output_mode != :name_only && output_mode != :name_status
+              yield "Common subdirectories: #{new_path}\n"
+            end
+
+          when :directory_to_file
+            next if diff_filter && diff_filter !~ /T/
+            if output_mode == :name_only
+              yield "#{new_path}\n"
+            elsif output_mode == :name_status
+              yield "T\t#{new_path}\n"
             else
-              Chef::ChefFS::FileSystem.child_pairs(old_entry, new_entry).each do |old_child,new_child|
-                diff_entries(old_child, new_child,
-                             recurse_depth ? recurse_depth - 1 : nil, output_mode) do |diff|
-                  yield diff
-                end
-              end
+              yield "File #{old_path} is a directory while file #{new_path} is a regular file\n"
             end
 
-        # If old is a directory and new is a file
-          elsif new_entry.exists?
+          when :file_to_directory
+            next if diff_filter && diff_filter !~ /T/
             if output_mode == :name_only
-              yield "#{new_entry.path_for_printing}\n"
+              yield "#{new_path}\n"
             elsif output_mode == :name_status
-              yield "T\t#{new_entry.path_for_printing}\n"
+              yield "T\t#{new_path}\n"
             else
-              yield "File #{new_entry.path_for_printing} is a directory while file #{new_entry.path_for_printing} is a regular file\n"
+              yield "File #{old_path} is a regular file while file #{new_path} is a directory\n"
             end
 
-        # If old is a directory and new does not exist
-          elsif new_entry.parent.can_have_child?(old_entry.name, old_entry.dir?)
+          when :deleted
+            next if diff_filter && diff_filter !~ /D/
             if output_mode == :name_only
-              yield "#{new_entry.path_for_printing}\n"
+              yield "#{new_path}\n"
             elsif output_mode == :name_status
-              yield "D\t#{new_entry.path_for_printing}\n"
+              yield "D\t#{new_path}\n"
+            elsif old_value
+              result = "diff --knife #{old_path} #{new_path}\n"
+              result << "deleted file\n"
+              result << diff_text(old_path, '/dev/null', old_value, '')
+              yield result
             else
-              yield "Only in #{old_entry.parent.path_for_printing}: #{old_entry.name}\n"
+              yield "Only in #{format_path.call(old_entry.parent)}: #{old_entry.name}\n"
             end
-          end
 
-        # If new is a directory and old is a file
-        elsif new_entry.dir?
-          if old_entry.exists?
+          when :added
+            next if diff_filter && diff_filter !~ /A/
             if output_mode == :name_only
-              yield "#{new_entry.path_for_printing}\n"
+              yield "#{new_path}\n"
             elsif output_mode == :name_status
-              yield "T\t#{new_entry.path_for_printing}\n"
+              yield "A\t#{new_path}\n"
+            elsif new_value
+              result = "diff --knife #{old_path} #{new_path}\n"
+              result << "new file\n"
+              result << diff_text('/dev/null', new_path, '', new_value)
+              yield result
             else
-              yield "File #{old_entry.path_for_printing} is a regular file while file #{old_entry.path_for_printing} is a directory\n"
+              yield "Only in #{format_path.call(new_entry.parent)}: #{new_entry.name}\n"
             end
 
-        # If new is a directory and old does not exist
-          elsif old_entry.parent.can_have_child?(new_entry.name, new_entry.dir?)
+          when :modified
+            next if diff_filter && diff_filter !~ /M/
             if output_mode == :name_only
-              yield "#{new_entry.path_for_printing}\n"
+              yield "#{new_path}\n"
             elsif output_mode == :name_status
-              yield "A\t#{new_entry.path_for_printing}\n"
+              yield "M\t#{new_path}\n"
+            else
+              result = "diff --knife #{old_path} #{new_path}\n"
+              result << diff_text(old_path, new_path, old_value, new_value)
+              yield result
+            end
+
+          when :both_nonexistent
+          when :added_cannot_upload
+          when :deleted_cannot_download
+          when :same
+            # Skip these silently
+          when :error
+            if error.is_a?(Chef::ChefFS::FileSystem::OperationFailedError)
+              ui.error "#{format_path.call(error.entry)} failed to #{error.operation}: #{error.message}" if ui
+              error = true
+            elsif error.is_a?(Chef::ChefFS::FileSystem::OperationNotAllowedError)
+              ui.error "#{format_path.call(error.entry)} #{error.reason}." if ui
             else
-              yield "Only in #{new_entry.parent.path_for_printing}: #{new_entry.name}\n"
+              raise error
             end
           end
+        end
+        if !found_match
+          ui.error "#{pattern}: No such file or directory on remote or local" if ui
+          error = true
+        end
+        error
+      end
+
+      def self.diff(pattern, old_root, new_root, recurse_depth, get_content)
+        Chef::ChefFS::Parallelizer.parallelize(Chef::ChefFS::FileSystem.list_pairs(pattern, old_root, new_root), :flatten => true) do |old_entry, new_entry|
+          diff_entries(old_entry, new_entry, recurse_depth, get_content)
+        end
+      end
+
+      # Diff two known entries (could be files or dirs)
+      def self.diff_entries(old_entry, new_entry, recurse_depth, get_content)
+        # If both are directories
+        if old_entry.dir?
+          if new_entry.dir?
+            if recurse_depth == 0
+              return [ [ :common_subdirectories, old_entry, new_entry ] ]
+            else
+              return Chef::ChefFS::Parallelizer.parallelize(Chef::ChefFS::FileSystem.child_pairs(old_entry, new_entry), :flatten => true) do |old_child, new_child|
+                Chef::ChefFS::CommandLine.diff_entries(old_child, new_child, recurse_depth ? recurse_depth - 1 : nil, get_content)
+              end
+            end
+
+          # If old is a directory and new is a file
+          elsif new_entry.exists?
+            return [ [ :directory_to_file, old_entry, new_entry ] ]
+
+          # If old is a directory and new does not exist
+          elsif new_entry.parent.can_have_child?(old_entry.name, old_entry.dir?)
+            return [ [ :deleted, old_entry, new_entry ] ]
+
+          # If the new entry does not and *cannot* exist, report that.
+          else
+            return [ [ :new_cannot_upload, old_entry, new_entry ] ]
+          end
+
+        # If new is a directory and old is a file
+        elsif new_entry.dir?
+          if old_entry.exists?
+            return [ [ :file_to_directory, old_entry, new_entry ] ]
+
+          # If new is a directory and old does not exist
+          elsif old_entry.parent.can_have_child?(new_entry.name, new_entry.dir?)
+            return [ [ :added, old_entry, new_entry ] ]
+
+          # If the new entry does not and *cannot* exist, report that.
+          else
+            return [ [ :old_cannot_upload, old_entry, new_entry ] ]
+          end
 
         # Neither is a directory, so they are diffable with file diff
         else
           are_same, old_value, new_value = Chef::ChefFS::FileSystem.compare(old_entry, new_entry)
           if are_same
-            return old_value != :none
+            if old_value == :none
+              return [ [ :both_nonexistent, old_entry, new_entry ] ]
+            else
+              return [ [ :same, old_entry, new_entry ] ]
+            end
           else
             if old_value == :none
               old_exists = false
@@ -108,6 +190,7 @@ class Chef
             else
               old_exists = true
             end
+
             if new_value == :none
               new_exists = false
             elsif new_value.nil?
@@ -119,24 +202,14 @@ class Chef
             # If one of the files doesn't exist, we only want to print the diff if the
             # other file *could be uploaded/downloaded*.
             if !old_exists && !old_entry.parent.can_have_child?(new_entry.name, new_entry.dir?)
-              return true
+              return [ [ :old_cannot_upload, old_entry, new_entry ] ]
             end
             if !new_exists && !new_entry.parent.can_have_child?(old_entry.name, old_entry.dir?)
-              return true
+              return [ [ :new_cannot_upload, old_entry, new_entry ] ]
             end
 
-            if output_mode == :name_only
-              yield "#{new_entry.path_for_printing}\n"
-            elsif output_mode == :name_status
-              if old_value == :none || (old_value == nil && !old_entry.exists?)
-                yield "A\t#{new_entry.path_for_printing}\n"
-              elsif new_value == :none
-                yield "D\t#{new_entry.path_for_printing}\n"
-              else
-                yield "M\t#{new_entry.path_for_printing}\n"
-              end
-            else
-              # If we haven't read the values yet, get them now.
+            if get_content
+              # If we haven't read the values yet, get them now so that they can be diffed
               begin
                 old_value = old_entry.read if old_value.nil?
               rescue Chef::ChefFS::FileSystem::NotFoundError
@@ -147,27 +220,19 @@ class Chef
               rescue Chef::ChefFS::FileSystem::NotFoundError
                 new_value = :none
               end
+            end
 
-              old_path = old_entry.path_for_printing
-              new_path = new_entry.path_for_printing
-              result = ''
-              result << "diff --knife #{old_path} #{new_path}\n"
-              if old_value == :none
-                result << "new file\n"
-                old_path = "/dev/null"
-                old_value = ''
-              end
-              if new_value == :none
-                result << "deleted file\n"
-                new_path = "/dev/null"
-                new_value = ''
-              end
-              result << diff_text(old_path, new_path, old_value, new_value)
-              yield result
+            if old_value == :none || (old_value == nil && !old_entry.exists?)
+              return [ [ :added, old_entry, new_entry, old_value, new_value ] ]
+            elsif new_value == :none
+              return [ [ :deleted, old_entry, new_entry, old_value, new_value ] ]
+            else
+              return [ [ :modified, old_entry, new_entry, old_value, new_value ] ]
             end
           end
         end
-        return true
+      rescue Chef::ChefFS::FileSystem::FileSystemError => e
+        return [ [ :error, old_entry, new_entry, nil, nil, e ] ]
       end
 
       private
@@ -191,19 +256,6 @@ class Chef
       end
 
       def self.diff_text(old_path, new_path, old_value, new_value)
-        # Reformat JSON for a nicer diff.
-        if old_path =~ /\.json$/
-          begin
-            reformatted_old_value = canonicalize_json(old_value)
-            reformatted_new_value = canonicalize_json(new_value)
-            old_value = reformatted_old_value
-            new_value = reformatted_new_value
-          rescue
-            # If JSON parsing fails, we just won't change any values and fall back
-            # to normal diff.
-          end
-        end
-
         # Copy to tempfiles before diffing
         # TODO don't copy things that are already in files!  Or find an in-memory diff algorithm
         begin
diff --git a/lib/chef/chef_fs/config.rb b/lib/chef/chef_fs/config.rb
new file mode 100644
index 0000000000..ab4cea89f2
--- /dev/null
+++ b/lib/chef/chef_fs/config.rb
@@ -0,0 +1,205 @@
+#
+# Author:: John Keiser (<jkeiser@opscode.com>)
+# Copyright:: Copyright (c) 2012 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/log'
+require 'chef/chef_fs/path_utils'
+
+class Chef
+  module ChefFS
+    #
+    # Helpers to take Chef::Config and create chef_fs and local_fs from it
+    #
+    class Config
+      def initialize(chef_config = Chef::Config, cwd = Dir.pwd)
+        @chef_config = chef_config
+        @cwd = cwd
+        configure_repo_paths
+      end
+
+      PATH_VARIABLES = %w(acl_path client_path cookbook_path container_path data_bag_path environment_path group_path node_path role_path user_path)
+
+      def chef_fs
+        @chef_fs ||= create_chef_fs
+      end
+
+      def create_chef_fs
+        require 'chef/chef_fs/file_system/chef_server_root_dir'
+        Chef::ChefFS::FileSystem::ChefServerRootDir.new("remote", @chef_config)
+      end
+
+      def local_fs
+        @local_fs ||= create_local_fs
+      end
+
+      def create_local_fs
+        require 'chef/chef_fs/file_system/chef_repository_file_system_root_dir'
+        Chef::ChefFS::FileSystem::ChefRepositoryFileSystemRootDir.new(object_paths)
+      end
+
+      # Returns the given real path's location relative to the server root.
+      #
+      # If chef_repo is /home/jkeiser/chef_repo,
+      # and pwd is /home/jkeiser/chef_repo/cookbooks,
+      # server_path('blah') == '/cookbooks/blah'
+      # server_path('../roles/blah.json') == '/roles/blah'
+      # server_path('../../readme.txt') == nil
+      # server_path('*/*ab*') == '/cookbooks/*/*ab*'
+      # server_path('/home/jkeiser/chef_repo/cookbooks/blah') == '/cookbooks/blah'
+      # server_path('/home/*/chef_repo/cookbooks/blah') == nil
+      #
+      # If there are multiple paths (cookbooks, roles, data bags, etc. can all
+      # have separate paths), and cwd+the path reaches into one of them, we will
+      # return a path relative to that.  Otherwise we will return a path to
+      # chef_repo.
+      #
+      # Globs are allowed as well, but globs outside server paths are NOT
+      # (presently) supported.  See above examples.  TODO support that.
+      #
+      # If the path does not reach into ANY specified directory, nil is returned.
+      def server_path(file_path)
+        pwd = File.expand_path(Dir.pwd)
+        absolute_path = Chef::ChefFS::PathUtils.realest_path(File.expand_path(file_path, pwd))
+
+        # Check all object paths (cookbooks_dir, data_bags_dir, etc.)
+        object_paths.each_pair do |name, paths|
+          paths.each do |path|
+            realest_path = Chef::ChefFS::PathUtils.realest_path(path)
+            if absolute_path[0,realest_path.length] == realest_path &&
+              (absolute_path.length == realest_path.length ||
+                absolute_path[realest_path.length,1] =~ /#{PathUtils.regexp_path_separator}/)
+              relative_path = Chef::ChefFS::PathUtils::relative_to(absolute_path, realest_path)
+              return relative_path == '.' ? "/#{name}" : "/#{name}/#{relative_path}"
+            end
+          end
+        end
+
+        # Check chef_repo_path
+        Array(@chef_config[:chef_repo_path]).flatten.each do |chef_repo_path|
+          realest_chef_repo_path = Chef::ChefFS::PathUtils.realest_path(chef_repo_path)
+          if absolute_path == realest_chef_repo_path
+            return '/'
+          end
+        end
+
+        nil
+      end
+
+      # The current directory, relative to server root
+      def base_path
+        @base_path ||= begin
+          if @chef_config[:chef_repo_path]
+            server_path(File.expand_path(@cwd))
+          else
+            nil
+          end
+        end
+      end
+
+      # Print the given server path, relative to the current directory
+      def format_path(entry)
+        server_path = entry.path
+        if base_path && server_path[0,base_path.length] == base_path
+          if server_path == base_path
+            return "."
+          elsif server_path[base_path.length,1] == "/"
+            return server_path[base_path.length + 1, server_path.length - base_path.length - 1]
+          elsif base_path == "/" && server_path[0,1] == "/"
+            return server_path[1, server_path.length - 1]
+          end
+        end
+        server_path
+      end
+
+      def require_chef_repo_path
+        if !@chef_config[:chef_repo_path]
+          Chef::Log.error("Must specify either chef_repo_path or cookbook_path in Chef config file")
+          exit(1)
+        end
+      end
+
+      private
+
+      def object_paths
+        @object_paths ||= begin
+          require_chef_repo_path
+
+          result = {}
+          case @chef_config[:repo_mode]
+          when 'static'
+            object_names = %w(cookbooks data_bags environments roles)
+          when 'hosted_everything'
+            object_names = %w(acls clients cookbooks containers data_bags environments groups nodes roles)
+          else
+            object_names = %w(clients cookbooks data_bags environments nodes roles users)
+          end
+          object_names.each do |object_name|
+            variable_name = "#{object_name[0..-2]}_path" # cookbooks -> cookbook_path
+            paths = Array(@chef_config[variable_name]).flatten
+            result[object_name] = paths.map { |path| File.expand_path(path) }
+          end
+          result
+        end
+      end
+
+      def configure_repo_paths
+        # Smooth out some (for now) inappropriate defaults set by Chef
+        if @chef_config[:cookbook_path] == [ @chef_config.platform_specific_path("/var/chef/cookbooks"),
+                      @chef_config.platform_specific_path("/var/chef/site-cookbooks") ]
+          @chef_config[:cookbook_path] = nil
+        end
+        if @chef_config[:data_bag_path] == @chef_config.platform_specific_path('/var/chef/data_bags')
+          @chef_config[:data_bag_path] = nil
+        end
+        if @chef_config[:node_path] == '/var/chef/node'
+          @chef_config[:node_path] = nil
+        end
+        if @chef_config[:role_path] == @chef_config.platform_specific_path('/var/chef/roles')
+          @chef_config[:role_path] = nil
+        end
+
+        # Infer chef_repo_path from cookbook_path if not speciifed
+        if !@chef_config[:chef_repo_path]
+          if @chef_config[:cookbook_path]
+            @chef_config[:chef_repo_path] = Array(@chef_config[:cookbook_path]).flatten.map { |path| File.expand_path('..', path) }
+          end
+        end
+
+        # Default to getting *everything* from the server.
+        if !@chef_config[:repo_mode]
+          if @chef_config[:chef_server_url] =~ /\/+organizations\/.+/
+            @chef_config[:repo_mode] = 'hosted_everything'
+          else
+            @chef_config[:repo_mode] = 'everything'
+          end
+        end
+
+        # Infer any *_path variables that are not specified
+        if @chef_config[:chef_repo_path]
+          PATH_VARIABLES.each do |variable_name|
+            chef_repo_paths = Array(@chef_config[:chef_repo_path]).flatten
+            variable = variable_name.to_sym
+            if !@chef_config[variable]
+              # cookbook_path -> cookbooks
+              @chef_config[variable] = chef_repo_paths.map { |path| File.join(path, "#{variable_name[0..-6]}s") }
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/chef/chef_fs/data_handler/acl_data_handler.rb b/lib/chef/chef_fs/data_handler/acl_data_handler.rb
new file mode 100644
index 0000000000..64fed7cac6
--- /dev/null
+++ b/lib/chef/chef_fs/data_handler/acl_data_handler.rb
@@ -0,0 +1,26 @@
+require 'chef/chef_fs/data_handler/data_handler_base'
+
+class Chef
+  module ChefFS
+    module DataHandler
+      class AclDataHandler < DataHandlerBase
+        def normalize(node, entry)
+          # Normalize the order of the keys for easier reading
+          result = super(node, {
+            'create' => {},
+            'read' => {},
+            'update' => {},
+            'delete' => {},
+            'grant' => {}
+            })
+          result.keys.each do |key|
+            result[key] = super(result[key], { 'actors' => [], 'groups' => [] })
+            result[key]['actors'] = result[key]['actors'].sort
+            result[key]['groups'] = result[key]['groups'].sort
+          end
+          result
+        end
+      end
+    end
+  end
+end
diff --git a/lib/chef/chef_fs/data_handler/client_data_handler.rb b/lib/chef/chef_fs/data_handler/client_data_handler.rb
new file mode 100644
index 0000000000..a92e486782
--- /dev/null
+++ b/lib/chef/chef_fs/data_handler/client_data_handler.rb
@@ -0,0 +1,38 @@
+require 'chef/chef_fs/data_handler/data_handler_base'
+require 'chef/api_client'
+
+class Chef
+  module ChefFS
+    module DataHandler
+      class ClientDataHandler < DataHandlerBase
+        def normalize(client, entry)
+          defaults = {
+            'name' => remove_dot_json(entry.name),
+            'clientname' => remove_dot_json(entry.name),
+            'orgname' => entry.org,
+            'admin' => false,
+            'validator' => false,
+            'chef_type' => 'client'
+          }
+          if entry.org
+            defaults['orgname'] = entry.org
+          end
+          result = super(client, defaults)
+          # You can NOT send json_class, or it will fail
+          result.delete('json_class')
+          result
+        end
+
+        def preserve_key(key)
+          return key == 'name'
+        end
+
+        def chef_class
+          Chef::ApiClient
+        end
+
+        # There is no Ruby API for Chef::ApiClient
+      end
+    end
+  end
+end
diff --git a/lib/chef/chef_fs/data_handler/container_data_handler.rb b/lib/chef/chef_fs/data_handler/container_data_handler.rb
new file mode 100644
index 0000000000..714f83824a
--- /dev/null
+++ b/lib/chef/chef_fs/data_handler/container_data_handler.rb
@@ -0,0 +1,29 @@
+require 'chef/chef_fs/data_handler/data_handler_base'
+
+class Chef
+  module ChefFS
+    module DataHandler
+      class ContainerDataHandler < DataHandlerBase
+        def normalize(container, entry)
+          super(container, {
+            'containername' => remove_dot_json(entry.name),
+            'containerpath' => remove_dot_json(entry.name)
+          })
+        end
+
+        def preserve_key(key)
+          return key == 'containername'
+        end
+
+        def verify_integrity(object, entry, &on_error)
+          base_name = remove_dot_json(entry.name)
+          if object['containername'] != base_name
+            on_error.call("Name in #{entry.path_for_printing} must be '#{base_name}' (is '#{object['name']}')")
+          end
+        end
+
+        # There is no chef_class for users, nor does to_ruby work.
+      end
+    end
+  end
+end
diff --git a/lib/chef/chef_fs/data_handler/cookbook_data_handler.rb b/lib/chef/chef_fs/data_handler/cookbook_data_handler.rb
new file mode 100644
index 0000000000..d0333db48f
--- /dev/null
+++ b/lib/chef/chef_fs/data_handler/cookbook_data_handler.rb
@@ -0,0 +1,38 @@
+require 'chef/chef_fs/data_handler/data_handler_base'
+require 'chef/cookbook/metadata'
+
+class Chef
+  module ChefFS
+    module DataHandler
+      class CookbookDataHandler < DataHandlerBase
+        def normalize(cookbook, entry)
+          version = entry.name
+          name = entry.parent.name
+          result = super(cookbook, {
+            'name' => "#{name}-#{version}",
+            'version' => version,
+            'cookbook_name' => name,
+            'json_class' => 'Chef::CookbookVersion',
+            'chef_type' => 'cookbook_version',
+            'frozen?' => false,
+            'metadata' => {}
+          })
+          result['metadata'] = super(result['metadata'], {
+            'version' => version,
+            'name' => name
+          })
+        end
+
+        def preserve_key(key)
+          return key == 'cookbook_name' || key == 'version'
+        end
+
+        def chef_class
+          Chef::Cookbook::Metadata
+        end
+
+        # Not using this yet, so not sure if to_ruby will be useful.
+      end
+    end
+  end
+end
diff --git a/lib/chef/chef_fs/data_handler/data_bag_item_data_handler.rb b/lib/chef/chef_fs/data_handler/data_bag_item_data_handler.rb
new file mode 100644
index 0000000000..1f466ec5ac
--- /dev/null
+++ b/lib/chef/chef_fs/data_handler/data_bag_item_data_handler.rb
@@ -0,0 +1,55 @@
+require 'chef/chef_fs/data_handler/data_handler_base'
+require 'chef/data_bag_item'
+
+class Chef
+  module ChefFS
+    module DataHandler
+      class DataBagItemDataHandler < DataHandlerBase
+        def normalize(data_bag_item, entry)
+          # If it's wrapped with raw_data, unwrap it.
+          if data_bag_item['json_class'] == 'Chef::DataBagItem' && data_bag_item['raw_data']
+            data_bag_item = data_bag_item['raw_data']
+          end
+          # chef_type and data_bag only come back from PUT and POST, but we'll
+          # normalize them in in case someone is comparing with those results.
+          super(data_bag_item, {
+            'chef_type' => 'data_bag_item',
+            'data_bag' => entry.parent.name,
+            'id' => remove_dot_json(entry.name)
+          })
+        end
+
+        def normalize_for_post(data_bag_item, entry)
+          {
+            "name" => "data_bag_item_#{entry.parent.name}_#{remove_dot_json(entry.name)}",
+            "json_class" => "Chef::DataBagItem",
+            "chef_type" => "data_bag_item",
+            "data_bag" => entry.parent.name,
+            "raw_data" => normalize(data_bag_item, entry)
+          }
+        end
+
+        def normalize_for_put(data_bag_item, entry)
+          normalize_for_post(data_bag_item, entry)
+        end
+
+        def preserve_key(key)
+          return key == 'id'
+        end
+
+        def chef_class
+          Chef::DataBagItem
+        end
+
+        def verify_integrity(object, entry, &on_error)
+          base_name = remove_dot_json(entry.name)
+          if object['raw_data']['id'] != base_name
+            on_error.call("ID in #{entry.path_for_printing} must be '#{base_name}' (is '#{object['raw_data']['id']}')")
+          end
+        end
+
+        # Data bags do not support .rb files (or if they do, it's undocumented)
+      end
+    end
+  end
+end
diff --git a/lib/chef/chef_fs/data_handler/data_handler_base.rb b/lib/chef/chef_fs/data_handler/data_handler_base.rb
new file mode 100644
index 0000000000..11e5bae31c
--- /dev/null
+++ b/lib/chef/chef_fs/data_handler/data_handler_base.rb
@@ -0,0 +1,128 @@
+class Chef
+  module ChefFS
+    module DataHandler
+      class DataHandlerBase
+        def minimize(object, entry)
+          default_object = default(entry)
+          object.each_pair do |key, value|
+            if default_object[key] == value && !preserve_key(key)
+              object.delete(key)
+            end
+          end
+          object
+        end
+
+        def remove_dot_json(name)
+          if name.length < 5 || name[-5,5] != ".json"
+            raise "Invalid name #{path}: must end in .json"
+          end
+          name[0,name.length-5]
+        end
+
+        def preserve_key(key)
+          false
+        end
+
+        def default(entry)
+          normalize({}, entry)
+        end
+
+        def normalize(object, defaults)
+          # Make a normalized result in the specified order for diffing
+          result = {}
+          defaults.each_pair do |key, default|
+            result[key] = object.has_key?(key) ? object[key] : default
+          end
+          object.each_pair do |key, value|
+            result[key] = value if !result.has_key?(key)
+          end
+          result
+        end
+
+        def normalize_for_post(object, entry)
+          normalize(object, entry)
+        end
+
+        def normalize_for_put(object, entry)
+          normalize(object, entry)
+        end
+
+        def normalize_run_list(run_list)
+          run_list.map{|item|
+            case item.to_s
+            when /^recipe\[.*\]$/
+              item # explicit recipe
+            when /^role\[.*\]$/
+              item # explicit role
+            else
+              "recipe[#{item}]"
+            end
+          }.uniq
+        end
+
+        def from_ruby(ruby)
+          chef_class.from_file(ruby).to_hash
+        end
+
+        def chef_object(object)
+          chef_class.json_create(object)
+        end
+
+        def to_ruby(object)
+          raise NotImplementedError
+        end
+
+        def chef_class
+          raise NotImplementedError
+        end
+
+        def to_ruby_keys(object, keys)
+          result = ''
+          keys.each do |key|
+            if object[key]
+              if object[key].is_a?(Hash)
+                if object[key].size > 0
+                  result << key
+                  first = true
+                  object[key].each_pair do |k,v|
+                    if first
+                      first = false
+                    else
+                      result << ' '*key.length
+                    end
+                    result << " #{k.inspect} => #{v.inspect}\n"
+                  end
+                end
+              elsif object[key].is_a?(Array)
+                if object[key].size > 0
+                  result << key
+                  first = true
+                  object[key].each do |value|
+                    if first
+                      first = false
+                    else
+                      result << ", "
+                    end
+                    result << value.inspect
+                  end
+                  result << "\n"
+                end
+              elsif !object[key].nil?
+                result << "#{key} #{object[key].inspect}\n"
+              end
+            end
+          end
+          result
+        end
+
+        def verify_integrity(object, entry, &on_error)
+          base_name = remove_dot_json(entry.name)
+          if object['name'] != base_name
+            on_error.call("Name must be '#{base_name}' (is '#{object['name']}')")
+          end
+        end
+
+      end # class DataHandlerBase
+    end
+  end
+end
diff --git a/lib/chef/chef_fs/data_handler/environment_data_handler.rb b/lib/chef/chef_fs/data_handler/environment_data_handler.rb
new file mode 100644
index 0000000000..6b13615968
--- /dev/null
+++ b/lib/chef/chef_fs/data_handler/environment_data_handler.rb
@@ -0,0 +1,40 @@
+require 'chef/chef_fs/data_handler/data_handler_base'
+require 'chef/environment'
+
+class Chef
+  module ChefFS
+    module DataHandler
+      class EnvironmentDataHandler < DataHandlerBase
+        def normalize(environment, entry)
+          super(environment, {
+            'name' => remove_dot_json(entry.name),
+            'description' => '',
+            'cookbook_versions' => {},
+            'default_attributes' => {},
+            'override_attributes' => {},
+            'json_class' => 'Chef::Environment',
+            'chef_type' => 'environment'
+          })
+        end
+
+        def preserve_key(key)
+          return key == 'name'
+        end
+
+        def chef_class
+          Chef::Environment
+        end
+
+        def to_ruby(object)
+          result = to_ruby_keys(object, %w(name description default_attributes override_attributes))
+          if object['cookbook_versions']
+            object['cookbook_versions'].each_pair do |name, version|
+              result << "cookbook #{name.inspect}, #{version.inspect}"
+            end
+          end
+          result
+        end
+      end
+    end
+  end
+end
diff --git a/lib/chef/chef_fs/data_handler/group_data_handler.rb b/lib/chef/chef_fs/data_handler/group_data_handler.rb
new file mode 100644
index 0000000000..b27bf87c50
--- /dev/null
+++ b/lib/chef/chef_fs/data_handler/group_data_handler.rb
@@ -0,0 +1,51 @@
+require 'chef/chef_fs/data_handler/data_handler_base'
+require 'chef/api_client'
+
+class Chef
+  module ChefFS
+    module DataHandler
+      class GroupDataHandler < DataHandlerBase
+        def normalize(group, entry)
+          defaults = {
+            'name' => remove_dot_json(entry.name),
+            'groupname' => remove_dot_json(entry.name),
+            'users' => [],
+            'clients' => [],
+            'groups' => [],
+          }
+          if entry.org
+            defaults['orgname'] = entry.org
+          end
+          result = super(group, defaults)
+          if result['actors'] && result['actors'].sort.uniq == (result['users'] + result['clients']).sort.uniq
+            result.delete('actors')
+          end
+          result
+        end
+
+        def normalize_for_put(group, entry)
+          result = super(group, entry)
+          result['actors'] = {
+            'users' => result['users'],
+            'clients' => result['clients'],
+            'groups' => result['groups']
+          }
+          result.delete('users')
+          result.delete('clients')
+          result.delete('groups')
+          result
+        end
+
+        def preserve_key(key)
+          return key == 'name'
+        end
+
+        def chef_class
+          Chef::ApiClient
+        end
+
+        # There is no Ruby API for Chef::ApiClient
+      end
+    end
+  end
+end
diff --git a/lib/chef/chef_fs/data_handler/node_data_handler.rb b/lib/chef/chef_fs/data_handler/node_data_handler.rb
new file mode 100644
index 0000000000..13e60e4fc1
--- /dev/null
+++ b/lib/chef/chef_fs/data_handler/node_data_handler.rb
@@ -0,0 +1,36 @@
+require 'chef/chef_fs/data_handler/data_handler_base'
+require 'chef/node'
+
+class Chef
+  module ChefFS
+    module DataHandler
+      class NodeDataHandler < DataHandlerBase
+        def normalize(node, entry)
+          result = super(node, {
+            'name' => remove_dot_json(entry.name),
+            'json_class' => 'Chef::Node',
+            'chef_type' => 'node',
+            'chef_environment' => '_default',
+            'override' => {},
+            'normal' => {},
+            'default' => {},
+            'automatic' => {},
+            'run_list' => []
+          })
+          result['run_list'] = normalize_run_list(result['run_list'])
+          result
+        end
+
+        def preserve_key(key)
+          return key == 'name'
+        end
+
+        def chef_class
+          Chef::Node
+        end
+
+        # Nodes do not support .rb files
+      end
+    end
+  end
+end
diff --git a/lib/chef/chef_fs/data_handler/role_data_handler.rb b/lib/chef/chef_fs/data_handler/role_data_handler.rb
new file mode 100644
index 0000000000..c803449862
--- /dev/null
+++ b/lib/chef/chef_fs/data_handler/role_data_handler.rb
@@ -0,0 +1,40 @@
+require 'chef/chef_fs/data_handler/data_handler_base'
+require 'chef/role'
+
+class Chef
+  module ChefFS
+    module DataHandler
+      class RoleDataHandler < DataHandlerBase
+        def normalize(role, entry)
+          result = super(role, {
+            'name' => remove_dot_json(entry.name),
+            'description' => '',
+            'json_class' => 'Chef::Role',
+            'chef_type' => 'role',
+            'default_attributes' => {},
+            'override_attributes' => {},
+            'run_list' => [],
+            'env_run_lists' => {}
+          })
+          result['run_list'] = normalize_run_list(result['run_list'])
+          result['env_run_lists'].each_pair do |env, run_list|
+            result['env_run_lists'][env] = normalize_run_list(run_list)
+          end
+          result
+        end
+
+        def preserve_key(key)
+          return key == 'name'
+        end
+
+        def chef_class
+          Chef::Role
+        end
+
+        def to_ruby(object)
+          to_ruby_keys(object, %w(name description default_attributes override_attributes run_list env_run_lists))
+        end
+      end
+    end
+  end
+end
diff --git a/lib/chef/chef_fs/data_handler/user_data_handler.rb b/lib/chef/chef_fs/data_handler/user_data_handler.rb
new file mode 100644
index 0000000000..7bbb87645b
--- /dev/null
+++ b/lib/chef/chef_fs/data_handler/user_data_handler.rb
@@ -0,0 +1,27 @@
+require 'chef/chef_fs/data_handler/data_handler_base'
+
+class Chef
+  module ChefFS
+    module DataHandler
+      class UserDataHandler < DataHandlerBase
+        def normalize(user, entry)
+          super(user, {
+            'name' => remove_dot_json(entry.name),
+            'admin' => false,
+            'json_class' => 'Chef::WebUIUser',
+            'chef_type' => 'webui_user',
+            'salt' => nil,
+            'password' => nil,
+            'openid' => nil
+          })
+        end
+
+        def preserve_key(key)
+          return key == 'name'
+        end
+
+        # There is no chef_class for users, nor does to_ruby work.
+      end
+    end
+  end
+end
diff --git a/lib/chef/chef_fs/file_system.rb b/lib/chef/chef_fs/file_system.rb
index 1805869e32..a6e14e548c 100644
--- a/lib/chef/chef_fs/file_system.rb
+++ b/lib/chef/chef_fs/file_system.rb
@@ -17,40 +17,63 @@
 #
 
 require 'chef/chef_fs/path_utils'
+require 'chef/chef_fs/file_system/default_environment_cannot_be_modified_error'
+require 'chef/chef_fs/file_system/operation_failed_error'
+require 'chef/chef_fs/file_system/operation_not_allowed_error'
+require 'chef/chef_fs/parallelizer'
 
 class Chef
   module ChefFS
     module FileSystem
-      # Yields a list of all things under (and including) this entry that match the
+      # Returns a list of all things under (and including) this entry that match the
       # given pattern.
       #
       # ==== Attributes
       #
-      # * +entry+ - Entry to start listing under
+      # * +root+ - Entry to start listing under
       # * +pattern+ - Chef::ChefFS::FilePattern to match children under
       #
-      def self.list(entry, pattern, &block)
-        # Include self in results if it matches
-        if pattern.match?(entry.path)
-          block.call(entry)
+      def self.list(root, pattern)
+        Lister.new(root, pattern)
+      end
+
+      class Lister
+        include Enumerable
+
+        def initialize(root, pattern)
+          @root = root
+          @pattern = pattern
         end
 
-        if entry.dir? && pattern.could_match_children?(entry.path)
-          # If it's possible that our children could match, descend in and add matches.
-          exact_child_name = pattern.exact_child_name_under(entry.path)
+        attr_reader :root
+        attr_reader :pattern
 
-          # If we've got an exact name, don't bother listing children; just grab the
-          # child with the given name.
-          if exact_child_name
-            exact_child = entry.child(exact_child_name)
-            if exact_child
-              list(exact_child, pattern, &block)
-            end
+        def each(&block)
+          list_from(root, &block)
+        end
 
-          # Otherwise, go through all children and find any matches
-          else
-            entry.children.each do |child|
-              list(child, pattern, &block)
+        def list_from(entry, &block)
+          # Include self in results if it matches
+          if pattern.match?(entry.path)
+            block.call(entry)
+          end
+
+          if pattern.could_match_children?(entry.path)
+            # If it's possible that our children could match, descend in and add matches.
+            exact_child_name = pattern.exact_child_name_under(entry.path)
+
+            # If we've got an exact name, don't bother listing children; just grab the
+            # child with the given name.
+            if exact_child_name
+              exact_child = entry.child(exact_child_name)
+              if exact_child
+                list_from(exact_child, &block)
+              end
+
+            # Otherwise, go through all children and find any matches
+            elsif entry.dir?
+              results = Parallelizer::parallelize(entry.children, :flatten => true) { |child| Chef::ChefFS::FileSystem.list(child, pattern) }
+              results.each(&block)
             end
           end
         end
@@ -113,16 +136,20 @@ class Chef
       #       puts message
       #     end
       #
-      def self.copy_to(pattern, src_root, dest_root, recurse_depth, options)
+      def self.copy_to(pattern, src_root, dest_root, recurse_depth, options, ui = nil, format_path = nil)
         found_result = false
-        list_pairs(pattern, src_root, dest_root) do |src, dest|
+        error = false
+        parallel_do(list_pairs(pattern, src_root, dest_root)) do |src, dest|
           found_result = true
-          new_dest_parent = get_or_create_parent(dest, options)
-          copy_entries(src, dest, new_dest_parent, recurse_depth, options)
+          new_dest_parent = get_or_create_parent(dest, options, ui, format_path)
+          child_error = copy_entries(src, dest, new_dest_parent, recurse_depth, options, ui, format_path)
+          error ||= child_error
         end
         if !found_result && pattern.exact_path
-          puts "#{pattern}: No such file or directory on remote or local"
+          ui.error "#{pattern}: No such file or directory on remote or local" if ui
+          error = true
         end
+        error
       end
 
       # Yield entries for children that are in either +a_root+ or +b_root+, with
@@ -136,26 +163,44 @@ class Chef
       #
       # ==== Example
       #
-      #     Chef::ChefFS::FileSystem.list_pairs(FilePattern.new('**x.txt', a_root, b_root)) do |a, b|
+      #     Chef::ChefFS::FileSystem.list_pairs(FilePattern.new('**x.txt', a_root, b_root)).each do |a, b|
       #       ...
       #     end
       #
       def self.list_pairs(pattern, a_root, b_root)
-        # Make sure everything on the server is also on the filesystem, and diff
-        found_paths = Set.new
-        Chef::ChefFS::FileSystem.list(a_root, pattern) do |a|
-          found_paths << a.path
-          b = Chef::ChefFS::FileSystem.resolve_path(b_root, a.path)
-          yield [ a, b ]
+        PairLister.new(pattern, a_root, b_root)
+      end
+
+      class PairLister
+        include Enumerable
+
+        def initialize(pattern, a_root, b_root)
+          @pattern = pattern
+          @a_root = a_root
+          @b_root = b_root
         end
 
-        # Check the outer regex pattern to see if it matches anything on the
-        # filesystem that isn't on the server
-        Chef::ChefFS::FileSystem.list(b_root, pattern) do |b|
-          if !found_paths.include?(b.path)
-            a = Chef::ChefFS::FileSystem.resolve_path(a_root, b.path)
+        attr_reader :pattern
+        attr_reader :a_root
+        attr_reader :b_root
+
+        def each
+          # Make sure everything on the server is also on the filesystem, and diff
+          found_paths = Set.new
+          Chef::ChefFS::FileSystem.list(a_root, pattern).each do |a|
+            found_paths << a.path
+            b = Chef::ChefFS::FileSystem.resolve_path(b_root, a.path)
             yield [ a, b ]
           end
+
+          # Check the outer regex pattern to see if it matches anything on the
+          # filesystem that isn't on the server
+          Chef::ChefFS::FileSystem.list(b_root, pattern).each do |b|
+            if !found_paths.include?(b.path)
+              a = Chef::ChefFS::FileSystem.resolve_path(a_root, b.path)
+              yield [ a, b ]
+            end
+          end
         end
       end
 
@@ -198,6 +243,7 @@ class Chef
           are_same, b_value, a_value = b.compare_to(a)
         end
         if are_same.nil?
+          # TODO these reads can be parallelized
           begin
             a_value = a.read if a_value.nil?
           rescue Chef::ChefFS::FileSystem::NotFoundError
@@ -216,7 +262,7 @@ class Chef
       private
 
       # Copy two entries (could be files or dirs)
-      def self.copy_entries(src_entry, dest_entry, new_dest_parent, recurse_depth, options)
+      def self.copy_entries(src_entry, dest_entry, new_dest_parent, recurse_depth, options, ui, format_path)
         # A NOTE about this algorithm:
         # There are cases where this algorithm does too many network requests.
         # knife upload with a specific filename will first check if the file
@@ -228,131 +274,153 @@ class Chef
         # exist.
         # Will need to decide how that works with checksums, though.
 
-        if !src_entry.exists?
-          if options[:purge]
-            # If we would not have uploaded it, we will not purge it.
-            if src_entry.parent.can_have_child?(dest_entry.name, dest_entry.dir?)
-              if options[:dry_run]
-                puts "Would delete #{dest_entry.path_for_printing}"
+        error = false
+        begin
+          dest_path = format_path.call(dest_entry) if ui
+          src_path = format_path.call(src_entry) if ui
+          if !src_entry.exists?
+            if options[:purge]
+              # If we would not have uploaded it, we will not purge it.
+              if src_entry.parent.can_have_child?(dest_entry.name, dest_entry.dir?)
+                if options[:dry_run]
+                  ui.output "Would delete #{dest_path}" if ui
+                else
+                  dest_entry.delete(true)
+                  ui.output "Deleted extra entry #{dest_path} (purge is on)" if ui
+                end
               else
-                dest_entry.delete(true)
-                puts "Deleted extra entry #{dest_entry.path_for_printing} (purge is on)"
+                Chef::Log.info("Not deleting extra entry #{dest_path} (purge is off)") if ui
               end
-            else
-              Chef::Log.info("Not deleting extra entry #{dest_entry.path_for_printing} (purge is off)")
             end
-          end
 
-        elsif !dest_entry.exists?
-          if new_dest_parent.can_have_child?(src_entry.name, src_entry.dir?)
-            # If the entry can do a copy directly from filesystem, do that.
-            if new_dest_parent.respond_to?(:create_child_from)
-              if options[:dry_run]
-                puts "Would create #{dest_entry.path_for_printing}"
-              else
-                new_dest_parent.create_child_from(src_entry)
-                puts "Created #{dest_entry.path_for_printing}"
+          elsif !dest_entry.exists?
+            if new_dest_parent.can_have_child?(src_entry.name, src_entry.dir?)
+              # If the entry can do a copy directly from filesystem, do that.
+              if new_dest_parent.respond_to?(:create_child_from)
+                if options[:dry_run]
+                  ui.output "Would create #{dest_path}" if ui
+                else
+                  new_dest_parent.create_child_from(src_entry)
+                  ui.output "Created #{dest_path}" if ui
+                end
+                return
               end
-              return
-            end
 
-            if src_entry.dir?
-              if options[:dry_run]
-                puts "Would create #{dest_entry.path_for_printing}"
-                new_dest_dir = new_dest_parent.child(src_entry.name)
-              else
-                new_dest_dir = new_dest_parent.create_child(src_entry.name, nil)
-                puts "Created #{dest_entry.path_for_printing}/"
-              end
-              # Directory creation is recursive.
-              if recurse_depth != 0
-                src_entry.children.each do |src_child|
-                  new_dest_child = new_dest_dir.child(src_child.name)
-                  copy_entries(src_child, new_dest_child, new_dest_dir, recurse_depth ? recurse_depth - 1 : recurse_depth, options)
+              if src_entry.dir?
+                if options[:dry_run]
+                  ui.output "Would create #{dest_path}" if ui
+                  new_dest_dir = new_dest_parent.child(src_entry.name)
+                else
+                  new_dest_dir = new_dest_parent.create_child(src_entry.name, nil)
+                  ui.output "Created #{dest_path}" if ui
+                end
+                # Directory creation is recursive.
+                if recurse_depth != 0
+                  parallel_do(src_entry.children) do |src_child|
+                    new_dest_child = new_dest_dir.child(src_child.name)
+                    child_error = copy_entries(src_child, new_dest_child, new_dest_dir, recurse_depth ? recurse_depth - 1 : recurse_depth, options, ui, format_path)
+                    error ||= child_error
+                  end
                 end
-              end
-            else
-              if options[:dry_run]
-                puts "Would create #{dest_entry.path_for_printing}"
               else
-                new_dest_parent.create_child(src_entry.name, src_entry.read)
-                puts "Created #{dest_entry.path_for_printing}"
+                if options[:dry_run]
+                  ui.output "Would create #{dest_path}" if ui
+                else
+                  new_dest_parent.create_child(src_entry.name, src_entry.read)
+                  ui.output "Created #{dest_path}" if ui
+                end
               end
             end
-          end
 
-        else
-          # Both exist.
-
-          # If the entry can do a copy directly, do that.
-          if dest_entry.respond_to?(:copy_from)
-            if options[:force] || compare(src_entry, dest_entry)[0] == false
-              if options[:dry_run]
-                puts "Would update #{dest_entry.path_for_printing}"
-              else
-                dest_entry.copy_from(src_entry)
-                puts "Updated #{dest_entry.path_for_printing}"
-              end
-            end
-            return
-          end
+          else
+            # Both exist.
 
-          # If they are different types, log an error.
-          if src_entry.dir?
-            if dest_entry.dir?
-              # If both are directories, recurse into their children
-              if recurse_depth != 0
-                child_pairs(src_entry, dest_entry).each do |src_child, dest_child|
-                  copy_entries(src_child, dest_child, dest_entry, recurse_depth ? recurse_depth - 1 : recurse_depth, options)
+            # If the entry can do a copy directly, do that.
+            if dest_entry.respond_to?(:copy_from)
+              if options[:force] || compare(src_entry, dest_entry)[0] == false
+                if options[:dry_run]
+                  ui.output "Would update #{dest_path}" if ui
+                else
+                  dest_entry.copy_from(src_entry, options)
+                  ui.output "Updated #{dest_path}" if ui
                 end
               end
-            else
-              # If they are different types.
-              Chef::Log.error("File #{dest_entry.path_for_printing} is a directory while file #{dest_entry.path_for_printing} is a regular file\n")
               return
             end
-          else
-            if dest_entry.dir?
-              Chef::Log.error("File #{dest_entry.path_for_printing} is a directory while file #{dest_entry.path_for_printing} is a regular file\n")
-              return
-            else
 
-              # Both are files!  Copy them unless we're sure they are the same.
-              if options[:force]
-                should_copy = true
-                src_value = nil
+            # If they are different types, log an error.
+            if src_entry.dir?
+              if dest_entry.dir?
+                # If both are directories, recurse into their children
+                if recurse_depth != 0
+                  parallel_do(child_pairs(src_entry, dest_entry)) do |src_child, dest_child|
+                    child_error = copy_entries(src_child, dest_child, dest_entry, recurse_depth ? recurse_depth - 1 : recurse_depth, options, ui, format_path)
+                    error ||= child_error
+                  end
+                end
               else
-                are_same, src_value, dest_value = compare(src_entry, dest_entry)
-                should_copy = !are_same
+                # If they are different types.
+                ui.error("File #{src_path} is a directory while file #{dest_path} is a regular file\n") if ui
+                return
               end
-              if should_copy
-                if options[:dry_run]
-                  puts "Would update #{dest_entry.path_for_printing}"
+            else
+              if dest_entry.dir?
+                ui.error("File #{src_path} is a directory while file #{dest_path} is a regular file\n") if ui
+                return
+              else
+
+                # Both are files!  Copy them unless we're sure they are the same.'
+                if options[:diff] == false
+                  should_copy = false
+                elsif options[:force]
+                  should_copy = true
+                  src_value = nil
                 else
-                  src_value = src_entry.read if src_value.nil?
-                  dest_entry.write(src_value)
-                  puts "Updated #{dest_entry.path_for_printing}"
+                  are_same, src_value, dest_value = compare(src_entry, dest_entry)
+                  should_copy = !are_same
+                end
+                if should_copy
+                  if options[:dry_run]
+                    ui.output "Would update #{dest_path}" if ui
+                  else
+                    src_value = src_entry.read if src_value.nil?
+                    dest_entry.write(src_value)
+                    ui.output "Updated #{dest_path}" if ui
+                  end
                 end
               end
             end
           end
+        rescue DefaultEnvironmentCannotBeModifiedError => e
+          ui.warn "#{format_path.call(e.entry)} #{e.reason}." if ui
+        rescue OperationFailedError => e
+          ui.error "#{format_path.call(e.entry)} failed to #{e.operation}: #{e.message}" if ui
+          error = true
+        rescue OperationNotAllowedError => e
+          ui.error "#{format_path.call(e.entry)} #{e.reason}." if ui
+          error = true
         end
+        error
       end
 
-      def self.get_or_create_parent(entry, options)
+      def self.get_or_create_parent(entry, options, ui, format_path)
         parent = entry.parent
         if parent && !parent.exists?
-          parent_parent = get_or_create_parent(entry.parent, options)
+          parent_path = format_path.call(parent) if ui
+          parent_parent = get_or_create_parent(entry.parent, options, ui, format_path)
           if options[:dry_run]
-            puts "Would create #{parent.path_for_printing}"
+            ui.output "Would create #{parent_path}" if ui
           else
-            parent = parent_parent.create_child(parent.name, true)
-            puts "Created #{parent.path_for_printing}"
+            parent = parent_parent.create_child(parent.name, nil)
+            ui.output "Created #{parent_path}" if ui
           end
         end
         return parent
       end
 
+      def self.parallel_do(enum, options = {}, &block)
+        Chef::ChefFS::Parallelizer.parallelize(enum, options, &block).to_a
+      end
     end
   end
 end
diff --git a/lib/chef/chef_fs/file_system/acl_dir.rb b/lib/chef/chef_fs/file_system/acl_dir.rb
new file mode 100644
index 0000000000..c2354d478d
--- /dev/null
+++ b/lib/chef/chef_fs/file_system/acl_dir.rb
@@ -0,0 +1,64 @@
+#
+# Author:: John Keiser (<jkeiser@opscode.com>)
+# Copyright:: Copyright (c) 2013 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/chef_fs/file_system/base_fs_dir'
+require 'chef/chef_fs/file_system/acl_entry'
+require 'chef/chef_fs/file_system/operation_not_allowed_error'
+
+class Chef
+  module ChefFS
+    module FileSystem
+      class AclDir < BaseFSDir
+        def api_path
+          parent.parent.child(name).api_path
+        end
+
+        def child(name)
+          result = @children.select { |child| child.name == name }.first if @children
+          result ||= can_have_child?(name, false) ?
+                     AclEntry.new(name, self) : NonexistentFSObject.new(name, self)
+        end
+
+        def can_have_child?(name, is_dir)
+          name =~ /\.json$/ && !is_dir
+        end
+
+        def children
+          if @children.nil?
+            # Grab the ACTUAL children (/nodes, /containers, etc.) and get their names
+            names = parent.parent.child(name).children.map { |child| child.dir? ? "#{child.name}.json" : child.name }
+            @children = names.map { |name| AclEntry.new(name, self, true) }
+          end
+          @children
+        end
+
+        def create_child(name, file_contents)
+          raise OperationNotAllowedError.new(:create_child, self), "ACLs can only be updated, and can only be created when the corresponding object is created."
+        end
+
+        def data_handler
+          parent.data_handler
+        end
+
+        def rest
+          parent.rest
+        end
+      end
+    end
+  end
+end
diff --git a/lib/chef/chef_fs/file_system/acl_entry.rb b/lib/chef/chef_fs/file_system/acl_entry.rb
new file mode 100644
index 0000000000..0be9076038
--- /dev/null
+++ b/lib/chef/chef_fs/file_system/acl_entry.rb
@@ -0,0 +1,58 @@
+#
+# Author:: John Keiser (<jkeiser@opscode.com>)
+# Copyright:: Copyright (c) 2013 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/chef_fs/file_system/rest_list_entry'
+require 'chef/chef_fs/file_system/not_found_error'
+require 'chef/chef_fs/file_system/operation_not_allowed_error'
+require 'chef/chef_fs/file_system/operation_failed_error'
+
+class Chef
+  module ChefFS
+    module FileSystem
+      class AclEntry < RestListEntry
+        PERMISSIONS = %w(create read update delete grant)
+
+        def api_path
+          "#{super}/_acl"
+        end
+
+        def delete(recurse)
+          raise Chef::ChefFS::FileSystem::OperationNotAllowedError.new(:delete, self, e), "ACLs cannot be deleted."
+        end
+
+        def write(file_contents)
+          # ACL writes are fun.
+          acls = data_handler.normalize(JSON.parse(file_contents, :create_additions => false), self)
+          PERMISSIONS.each do |permission|
+            begin
+              rest.put_rest("#{api_path}/#{permission}", { permission => acls[permission] })
+            rescue Timeout::Error => e
+              raise Chef::ChefFS::FileSystem::OperationFailedError.new(:write, self, e), "Timeout writing: #{e}"
+            rescue Net::HTTPServerException => e
+              if e.response.code == "404"
+                raise Chef::ChefFS::FileSystem::NotFoundError.new(self, e)
+              else
+                raise Chef::ChefFS::FileSystem::OperationFailedError.new(:write, self, e), "HTTP error writing: #{e}"
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/chef/chef_fs/file_system/acls_dir.rb b/lib/chef/chef_fs/file_system/acls_dir.rb
new file mode 100644
index 0000000000..938bf73fb2
--- /dev/null
+++ b/lib/chef/chef_fs/file_system/acls_dir.rb
@@ -0,0 +1,68 @@
+#
+# Author:: John Keiser (<jkeiser@opscode.com>)
+# Copyright:: Copyright (c) 2013 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/chef_fs/file_system/base_fs_dir'
+require 'chef/chef_fs/file_system/acl_dir'
+require 'chef/chef_fs/file_system/cookbooks_acl_dir'
+require 'chef/chef_fs/file_system/acl_entry'
+require 'chef/chef_fs/data_handler/acl_data_handler'
+
+class Chef
+  module ChefFS
+    module FileSystem
+      class AclsDir < BaseFSDir
+        ENTITY_TYPES = %w(clients containers cookbooks data_bags environments groups nodes roles) # we don't read sandboxes, so we don't read their acls
+
+        def initialize(parent)
+          super('acls', parent)
+        end
+
+        def data_handler
+          @data_handler ||= Chef::ChefFS::DataHandler::AclDataHandler.new
+        end
+
+        def api_path
+          parent.api_path
+        end
+
+        def can_have_child?(name, is_dir)
+          is_dir ? ENTITY_TYPES.include(name) : name == 'organization.json'
+        end
+
+        def children
+          if @children.nil?
+            @children = ENTITY_TYPES.map do |entity_type|
+              case entity_type
+              when 'cookbooks'
+                CookbooksAclDir.new(entity_type, self)
+              else
+                AclDir.new(entity_type, self)
+              end
+            end
+            @children << AclEntry.new('organization.json', self, true) # the org acl is retrieved as GET /organizations/ORGNAME/ANYTHINGATALL/_acl
+          end
+          @children
+        end
+
+        def rest
+          parent.rest
+        end
+      end
+    end
+  end
+end
diff --git a/lib/chef/chef_fs/file_system/already_exists_error.rb b/lib/chef/chef_fs/file_system/already_exists_error.rb
new file mode 100644
index 0000000000..bf8994fdf3
--- /dev/null
+++ b/lib/chef/chef_fs/file_system/already_exists_error.rb
@@ -0,0 +1,31 @@
+#
+# Author:: John Keiser (<jkeiser@opscode.com>)
+# Copyright:: Copyright (c) 2012 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/chef_fs/file_system/operation_failed_error'
+
+class Chef
+  module ChefFS
+    module FileSystem
+      class AlreadyExistsError < OperationFailedError
+        def initialize(operation, entry, cause = nil)
+          super(operation, entry, cause)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/chef/chef_fs/file_system/base_fs_object.rb b/lib/chef/chef_fs/file_system/base_fs_object.rb
index 855892fc89..43e6a513d7 100644
--- a/lib/chef/chef_fs/file_system/base_fs_object.rb
+++ b/lib/chef/chef_fs/file_system/base_fs_object.rb
@@ -17,6 +17,7 @@
 #
 
 require 'chef/chef_fs/path_utils'
+require 'chef/chef_fs/file_system/operation_not_allowed_error'
 
 class Chef
   module ChefFS
@@ -39,42 +40,6 @@ class Chef
         attr_reader :parent
         attr_reader :path
 
-        def root
-          parent ? parent.root : self
-        end
-
-        def path_for_printing
-          if parent
-            parent_path = parent.path_for_printing
-            if parent_path == '.'
-              name
-            else
-              Chef::ChefFS::PathUtils::join(parent.path_for_printing, name)
-            end
-          else
-            name
-          end
-        end
-
-        def dir?
-          false
-        end
-
-        def exists?
-          true
-        end
-
-        def child(name)
-          NonexistentFSObject.new(name, self)
-        end
-
-        # Override can_have_child? to report whether a given file *could* be added
-        # to this directory.  (Some directories can't have subdirs, some can only have .json
-        # files, etc.)
-        def can_have_child?(name, is_dir)
-          false
-        end
-
         # Override this if you have a special comparison algorithm that can tell
         # you whether this entry is the same as another--either a quicker or a
         # more reliable one.  Callers will use this to decide whether to upload,
@@ -109,13 +74,107 @@ class Chef
         #       are_same = (value == other_value)
         #     end
         def compare_to(other)
-          return nil
+          nil
+        end
+
+        # Override can_have_child? to report whether a given file *could* be added
+        # to this directory.  (Some directories can't have subdirs, some can only have .json
+        # files, etc.)
+        def can_have_child?(name, is_dir)
+          false
+        end
+
+        # Get a child of this entry with the given name.  This MUST always
+        # return a child, even if it is NonexistentFSObject.  Overriders should
+        # take caution not to do expensive network requests to get the list of
+        # children to fulfill this request, unless absolutely necessary here; it
+        # is intended as a quick way to traverse a hierarchy.
+        #
+        # For example, knife show /data_bags/x/y.json will call
+        # root.child('data_bags').child('x').child('y.json'), which can then
+        # directly perform a network request to retrieve the y.json data bag.  No
+        # network request was necessary to retrieve
+        def child(name)
+          NonexistentFSObject.new(name, self)
+        end
+
+        # Override children to report your *actual* list of children as an array.
+        def children
+          raise NotFoundError.new(self) if !exists?
+          []
+        end
+
+        # Expand this entry into a chef object (Chef::Role, ::Node, etc.)
+        def chef_object
+          raise NotFoundError.new(self) if !exists?
+          nil
+        end
+
+        # Create a child of this entry with the given name and contents.  If
+        # contents is nil, create a directory.
+        #
+        # NOTE: create_child_from is an optional method that can also be added to
+        # your entry class, and will be called without actually reading the
+        # file_contents.  This is used for knife upload /cookbooks/cookbookname.
+        def create_child(name, file_contents)
+          raise NotFoundError.new(self) if !exists?
+          raise OperationNotAllowedError.new(:create_child, self)
+        end
+
+        # Delete this item, possibly recursively.  Entries MUST NOT delete a
+        # directory unless recurse is true.
+        def delete(recurse)
+          raise NotFoundError.new(self) if !exists?
+          raise OperationNotAllowedError.new(:delete, self)
+        end
+
+        # Ask whether this entry is a directory.  If not, it is a file.
+        def dir?
+          false
+        end
+
+        # Ask whether this entry exists.
+        def exists?
+          true
+        end
+
+        # Printable path, generally used to distinguish paths in one root from
+        # paths in another.
+        def path_for_printing
+          if parent
+            parent_path = parent.path_for_printing
+            if parent_path == '.'
+              name
+            else
+              Chef::ChefFS::PathUtils::join(parent.path_for_printing, name)
+            end
+          else
+            name
+          end
+        end
+
+        def root
+          parent ? parent.root : self
+        end
+
+        # Read the contents of this file entry.
+        def read
+          raise NotFoundError.new(self) if !exists?
+          raise OperationNotAllowedError.new(:read, self)
+        end
+
+        # Write the contents of this file entry.
+        def write(file_contents)
+          raise NotFoundError.new(self) if !exists?
+          raise OperationNotAllowedError.new(:write, self)
         end
 
         # Important directory attributes: name, parent, path, root
         # Overridable attributes: dir?, child(name), path_for_printing
-        # Abstract: read, write, delete, children
-      end
+        # Abstract: read, write, delete, children, can_have_child?, create_child, compare_to
+      end # class BaseFsObject
     end
   end
 end
+
+require 'chef/chef_fs/file_system/nonexistent_fs_object'
diff --git a/lib/chef/chef_fs/file_system/chef_repository_file_system_cookbooks_dir.rb b/lib/chef/chef_fs/file_system/chef_repository_file_system_cookbooks_dir.rb
new file mode 100644
index 0000000000..863a59a8d1
--- /dev/null
+++ b/lib/chef/chef_fs/file_system/chef_repository_file_system_cookbooks_dir.rb
@@ -0,0 +1,56 @@
+#
+# Author:: John Keiser (<jkeiser@opscode.com>)
+# Copyright:: Copyright (c) 2013 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/chef_fs/file_system/chef_repository_file_system_entry'
+require 'chef/cookbook/chefignore'
+
+class Chef
+  module ChefFS
+    module FileSystem
+      class ChefRepositoryFileSystemCookbooksDir < ChefRepositoryFileSystemEntry
+        def initialize(name, parent, file_path)
+          super(name, parent, file_path)
+          begin
+            @chefignore = Chef::Cookbook::Chefignore.new(self.file_path)
+          rescue Errno::EISDIR
+          rescue Errno::EACCES
+            # Work around a bug in Chefignore when chefignore is a directory
+          end
+        end
+
+        attr_reader :chefignore
+
+        def ignore_empty_directories?
+          true
+        end
+
+        def ignored?(entry)
+          return true if !entry.dir?
+          return true if entry.name.start_with?('.')
+
+          result = super(entry)
+
+          if result
+            Chef::Log.warn("Cookbook '#{entry.name}' is empty or entirely chefignored at #{entry.path_for_printing}")
+          end
+          result
+        end
+      end
+    end
+  end
+end
diff --git a/lib/chef/chef_fs/file_system/chef_repository_file_system_data_bags_dir.rb b/lib/chef/chef_fs/file_system/chef_repository_file_system_data_bags_dir.rb
new file mode 100644
index 0000000000..ba5cc75f6a
--- /dev/null
+++ b/lib/chef/chef_fs/file_system/chef_repository_file_system_data_bags_dir.rb
@@ -0,0 +1,37 @@
+#
+# Author:: John Keiser (<jkeiser@opscode.com>)
+# Copyright:: Copyright (c) 2013 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/chef_fs/file_system/chef_repository_file_system_entry'
+require 'chef/chef_fs/data_handler/data_bag_item_data_handler'
+
+class Chef
+  module ChefFS
+    module FileSystem
+      class ChefRepositoryFileSystemDataBagsDir < ChefRepositoryFileSystemEntry
+        def initialize(name, parent, path = nil)
+          super(name, parent, path, Chef::ChefFS::DataHandler::DataBagItemDataHandler.new)
+        end
+
+        def ignored?(entry)
+          return true if entry.dir? && entry.name.start_with?('.')
+          super(entry)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/chef/chef_fs/file_system/chef_repository_file_system_entry.rb b/lib/chef/chef_fs/file_system/chef_repository_file_system_entry.rb
index 87d904e830..fd599696ac 100644
--- a/lib/chef/chef_fs/file_system/chef_repository_file_system_entry.rb
+++ b/lib/chef/chef_fs/file_system/chef_repository_file_system_entry.rb
@@ -1,5 +1,6 @@
 #
 # Author:: John Keiser (<jkeiser@opscode.com>)
+# Author:: Ho-Sheng Hsiao (<hosh@opscode.com>)
 # Copyright:: Copyright (c) 2012 Opscode, Inc.
 # License:: Apache License, Version 2.0
 #
@@ -17,90 +18,102 @@
 #
 
 require 'chef/chef_fs/file_system/file_system_entry'
-require 'chef/cookbook/chefignore'
 require 'chef/cookbook/cookbook_version_loader'
-require 'chef/node'
-require 'chef/role'
-require 'chef/environment'
-require 'chef/data_bag_item'
-require 'chef/client'
 
 class Chef
   module ChefFS
     module FileSystem
       # ChefRepositoryFileSystemEntry works just like FileSystemEntry,
-      # except it pretends files in /cookbooks/chefignore don't exist
-      # and it can inflate Chef objects
+      # except can inflate Chef objects
       class ChefRepositoryFileSystemEntry < FileSystemEntry
-        def initialize(name, parent, file_path = nil)
+        def initialize(name, parent, file_path = nil, data_handler = nil)
           super(name, parent, file_path)
-          # Load /cookbooks/chefignore
-          if name == "cookbooks" && path == "/cookbooks" # We check name first because it's a faster fail than path
-            @chefignore = Chef::Cookbook::Chefignore.new(self.file_path)
-          # If we are a cookbook or a cookbook subdirectory, empty directories
-          # underneath us are ignored (since they cannot be uploaded)
-          elsif parent && parent.name === "cookbooks" && parent.path == "/cookbooks"
-            @ignore_empty_directories = true
-          elsif parent && parent.ignore_empty_directories?
-            @ignore_empty_directories = true
-          end
+          @data_handler = data_handler
         end
 
-        attr_reader :chefignore
+        def chefignore
+          nil
+        end
 
         def ignore_empty_directories?
-          @ignore_empty_directories
+          parent.ignore_empty_directories?
+        end
+
+        def data_handler
+          @data_handler || parent.data_handler
         end
 
         def chef_object
           begin
-            if parent.path == "/cookbooks"
+            if parent.path == '/cookbooks'
               loader = Chef::Cookbook::CookbookVersionLoader.new(file_path, parent.chefignore)
+              # We need the canonical cookbook name if we are using versioned cookbooks, but we don't
+              # want to spend a lot of time adding code to the main Chef libraries
+              if Chef::Config[:versioned_cookbooks]
+
+                _canonical_name = canonical_cookbook_name(File.basename(file_path))
+                fail "When versioned_cookbooks mode is on, cookbook #{file_path} must match format <cookbook_name>-x.y.z"  unless _canonical_name
+
+                # KLUDGE: We shouldn't have to use instance_variable_set
+                loader.instance_variable_set(:@cookbook_name, _canonical_name)
+              end
+
               loader.load_cookbooks
               return loader.cookbook_version
             end
 
-            # Otherwise the information to inflate the object, is in the file (json_class).
-            return Chef::JSONCompat.from_json(read)
+            # Otherwise, inflate the file using the chosen JSON class (if any)
+            return data_handler.chef_object(JSON.parse(read, :create_additions => false))
           rescue
             Chef::Log.error("Could not read #{path_for_printing} into a Chef object: #{$!}")
           end
           nil
         end
 
-        def children
-          @children ||= Dir.entries(file_path).select { |entry| entry != '.' && entry != '..' && !ignored?(entry) }.
-                                               map { |entry| ChefRepositoryFileSystemEntry.new(entry, self) }
+        # Exposed as a class method so that it can be used elsewhere
+        def self.canonical_cookbook_name(entry_name)
+          name_match = Chef::ChefFS::FileSystem::CookbookDir::VALID_VERSIONED_COOKBOOK_NAME.match(entry_name)
+          return nil if name_match.nil?
+          return name_match[1]
         end
 
-        attr_reader :chefignore
+        def canonical_cookbook_name(entry_name)
+          self.class.canonical_cookbook_name(entry_name)
+        end
+
+        def children
+          @children ||=
+            Dir.entries(file_path).sort.
+                select { |entry| entry != '.' && entry != '..' }.
+                map { |entry| ChefRepositoryFileSystemEntry.new(entry, self) }.
+                select { |entry| !ignored?(entry) }
+        end
 
         private
 
-        def ignored?(child_name)
-          # empty directories inside a cookbook are ignored
-          if ignore_empty_directories?
-            child_path = PathUtils.join(file_path, child_name)
-            if File.directory?(child_path) && Dir.entries(child_path) == [ '.', '..' ]
+        def ignored?(child_entry)
+          if child_entry.dir?
+            # empty cookbooks and cookbook directories are ignored
+            if ignore_empty_directories? && child_entry.children.size == 0
               return true
             end
-          end
-
-          ignorer = self
-          begin
-            if ignorer.chefignore
-              # Grab the path from entry to child
-              path_to_child = child_name
-              child = self
-              while child != ignorer
-                path_to_child = PathUtils.join(child.name, path_to_child)
-                child = child.parent
+          else
+            ignorer = parent
+            begin
+              if ignorer.chefignore
+                # Grab the path from entry to child
+                path_to_child = child_entry.name
+                child = self
+                while child.parent != ignorer
+                  path_to_child = PathUtils.join(child.name, path_to_child)
+                  child = child.parent
+                end
+                # Check whether that relative path is ignored
+                return ignorer.chefignore.ignored?(path_to_child)
               end
-              # Check whether that relative path is ignored
-              return ignorer.chefignore.ignored?(path_to_child)
-            end
-            ignorer = ignorer.parent
-          end while ignorer
+              ignorer = ignorer.parent
+            end while ignorer
+          end
         end
 
       end
diff --git a/lib/chef/chef_fs/file_system/chef_repository_file_system_root_dir.rb b/lib/chef/chef_fs/file_system/chef_repository_file_system_root_dir.rb
index fdad68003c..93ea7c64c8 100644
--- a/lib/chef/chef_fs/file_system/chef_repository_file_system_root_dir.rb
+++ b/lib/chef/chef_fs/file_system/chef_repository_file_system_root_dir.rb
@@ -16,14 +16,112 @@
 # limitations under the License.
 #
 
+require 'chef/chef_fs/file_system/base_fs_dir'
 require 'chef/chef_fs/file_system/chef_repository_file_system_entry'
+require 'chef/chef_fs/file_system/chef_repository_file_system_cookbooks_dir'
+require 'chef/chef_fs/file_system/chef_repository_file_system_data_bags_dir'
+require 'chef/chef_fs/file_system/multiplexed_dir'
+require 'chef/chef_fs/data_handler/client_data_handler'
+require 'chef/chef_fs/data_handler/environment_data_handler'
+require 'chef/chef_fs/data_handler/node_data_handler'
+require 'chef/chef_fs/data_handler/role_data_handler'
+require 'chef/chef_fs/data_handler/user_data_handler'
+require 'chef/chef_fs/data_handler/group_data_handler'
+require 'chef/chef_fs/data_handler/container_data_handler'
+require 'chef/chef_fs/data_handler/acl_data_handler'
 
 class Chef
   module ChefFS
     module FileSystem
-      class ChefRepositoryFileSystemRootDir < ChefRepositoryFileSystemEntry
-        def initialize(file_path)
-          super("", nil, file_path)
+      class ChefRepositoryFileSystemRootDir < BaseFSDir
+        def initialize(child_paths)
+          super("", nil)
+          @child_paths = child_paths
+        end
+
+        attr_reader :child_paths
+
+        def children
+          @children ||= child_paths.keys.sort.map { |name| make_child_entry(name) }.select { |child| !child.nil? }
+        end
+
+        def can_have_child?(name, is_dir)
+          child_paths.has_key?(name) && is_dir
+        end
+
+        def create_child(name, file_contents = nil)
+          child_paths[name].each do |path|
+            Dir.mkdir(path)
+          end
+          make_child_entry(name)
+        end
+
+        def ignore_empty_directories?
+          false
+        end
+
+        def chefignore
+          nil
+        end
+
+        def json_class
+          nil
+        end
+
+        # Used to print out the filesystem
+        def fs_description
+          repo_path = File.dirname(child_paths['cookbooks'][0])
+          result = "repository at #{repo_path}\n"
+          if Chef::Config[:versioned_cookbooks]
+            result << "  Multiple versions per cookbook\n"
+          else
+            result << "  One version per cookbook\n"
+          end
+          child_paths.each_pair do |name, paths|
+            if paths.any? { |path| File.dirname(path) != repo_path }
+              result << "  #{name} at #{paths.join(', ')}\n"
+            end
+          end
+          result
+        end
+
+        private
+
+        def make_child_entry(name)
+          paths = child_paths[name].select do |path|
+            File.exists?(path)
+          end
+          if paths.size == 0
+            return nil
+          end
+          if name == 'cookbooks'
+            dirs = paths.map { |path| ChefRepositoryFileSystemCookbooksDir.new(name, self, path) }
+          elsif name == 'data_bags'
+            dirs = paths.map { |path| ChefRepositoryFileSystemDataBagsDir.new(name, self, path) }
+          else
+            data_handler = case name
+              when 'clients'
+                Chef::ChefFS::DataHandler::ClientDataHandler.new
+              when 'environments'
+                Chef::ChefFS::DataHandler::EnvironmentDataHandler.new
+              when 'nodes'
+                Chef::ChefFS::DataHandler::NodeDataHandler.new
+              when 'roles'
+                Chef::ChefFS::DataHandler::RoleDataHandler.new
+              when 'users'
+                Chef::ChefFS::DataHandler::UserDataHandler.new
+              when 'groups'
+                Chef::ChefFS::DataHandler::GroupDataHandler.new
+              when 'containers'
+                Chef::ChefFS::DataHandler::ContainerDataHandler.new
+              when 'acls'
+                Chef::ChefFS::DataHandler::AclDataHandler.new
+              else
+                raise "Unknown top level path #{name}"
+              end
+            dirs = paths.map { |path| ChefRepositoryFileSystemEntry.new(name, self, path, data_handler) }
+          end
+          MultiplexedDir.new(dirs)
         end
       end
     end
diff --git a/lib/chef/chef_fs/file_system/chef_server_root_dir.rb b/lib/chef/chef_fs/file_system/chef_server_root_dir.rb
index d3c217d11c..5eb72657c5 100644
--- a/lib/chef/chef_fs/file_system/chef_server_root_dir.rb
+++ b/lib/chef/chef_fs/file_system/chef_server_root_dir.rb
@@ -16,23 +16,31 @@
 # limitations under the License.
 #
 
+require 'chef/chef_fs/file_system/acls_dir'
 require 'chef/chef_fs/file_system/base_fs_dir'
 require 'chef/chef_fs/file_system/rest_list_dir'
 require 'chef/chef_fs/file_system/cookbooks_dir'
 require 'chef/chef_fs/file_system/data_bags_dir'
 require 'chef/chef_fs/file_system/nodes_dir'
+require 'chef/chef_fs/file_system/environments_dir'
+require 'chef/rest'
+require 'chef/chef_fs/data_handler/client_data_handler'
+require 'chef/chef_fs/data_handler/role_data_handler'
+require 'chef/chef_fs/data_handler/user_data_handler'
+require 'chef/chef_fs/data_handler/group_data_handler'
+require 'chef/chef_fs/data_handler/container_data_handler'
 
 class Chef
   module ChefFS
     module FileSystem
       class ChefServerRootDir < BaseFSDir
-        def initialize(root_name, chef_config, repo_mode)
+        def initialize(root_name, chef_config)
           super("", nil)
           @chef_server_url = chef_config[:chef_server_url]
           @chef_username = chef_config[:node_name]
           @chef_private_key = chef_config[:client_key]
           @environment = chef_config[:environment]
-          @repo_mode = repo_mode
+          @repo_mode = chef_config[:repo_mode]
           @root_name = root_name
         end
 
@@ -42,6 +50,10 @@ class Chef
         attr_reader :environment
         attr_reader :repo_mode
 
+        def fs_description
+          "Chef server at #{chef_server_url} (user #{chef_username}), repo_mode = #{repo_mode}"
+        end
+
         def rest
           Chef::REST.new(chef_server_url, chef_username, chef_private_key)
         end
@@ -58,26 +70,40 @@ class Chef
           is_dir && children.any? { |child| child.name == name }
         end
 
+        def org
+          @org ||= if URI.parse(chef_server_url).path =~ /^\/+organizations\/+([^\/]+)$/
+            $1
+          else
+            nil
+          end
+        end
+
         def children
           @children ||= begin
             result = [
               CookbooksDir.new(self),
               DataBagsDir.new(self),
-              RestListDir.new("environments", self),
-              RestListDir.new("roles", self)
+              EnvironmentsDir.new(self),
+              RestListDir.new("roles", self, nil, Chef::ChefFS::DataHandler::RoleDataHandler.new)
             ]
-            if repo_mode == 'everything'
+            if repo_mode == 'hosted_everything'
+              result += [
+                AclsDir.new(self),
+                RestListDir.new("clients", self, nil, Chef::ChefFS::DataHandler::ClientDataHandler.new),
+                RestListDir.new("containers", self, nil, Chef::ChefFS::DataHandler::ContainerDataHandler.new),
+                RestListDir.new("groups", self, nil, Chef::ChefFS::DataHandler::GroupDataHandler.new),
+                NodesDir.new(self)
+              ]
+            elsif repo_mode != 'static'
               result += [
-                RestListDir.new("clients", self),
+                RestListDir.new("clients", self, nil, Chef::ChefFS::DataHandler::ClientDataHandler.new),
                 NodesDir.new(self),
-                RestListDir.new("users", self)
+                RestListDir.new("users", self, nil, Chef::ChefFS::DataHandler::UserDataHandler.new)
               ]
             end
             result.sort_by { |child| child.name }
           end
         end
-
-        # Yeah, sorry, I'm not putting delete on this thing.
       end
     end
   end
diff --git a/lib/chef/chef_fs/file_system/cookbook_dir.rb b/lib/chef/chef_fs/file_system/cookbook_dir.rb
index e87d5dd49d..cae29a1690 100644
--- a/lib/chef/chef_fs/file_system/cookbook_dir.rb
+++ b/lib/chef/chef_fs/file_system/cookbook_dir.rb
@@ -27,12 +27,24 @@ class Chef
   module ChefFS
     module FileSystem
       class CookbookDir < BaseFSDir
-        def initialize(name, parent, versions = nil)
+        def initialize(name, parent, options = {})
           super(name, parent)
-          @versions = versions
+          @exists = options[:exists]
+          # If the name is apache2-1.0.0 and versioned_cookbooks is on, we know
+          # the actual cookbook_name and version.
+          if Chef::Config[:versioned_cookbooks]
+            if name =~ VALID_VERSIONED_COOKBOOK_NAME
+              @cookbook_name = $1
+              @version = $2
+            else
+              @exists = false
+            end
+          else
+            @cookbook_name = name
+          end
         end
 
-        attr_reader :versions
+        attr_reader :cookbook_name, :version
 
         COOKBOOK_SEGMENT_INFO = {
           :attributes => { :ruby_only => true },
@@ -46,12 +58,16 @@ class Chef
           :root_files => { }
         }
 
+        # See Erchef code
+        # https://github.com/opscode/chef_objects/blob/968a63344d38fd507f6ace05f73d53e9cd7fb043/src/chef_regex.erl#L94
+        VALID_VERSIONED_COOKBOOK_NAME = /^([.a-zA-Z0-9_-]+)-(\d+\.\d+\.\d+)$/
+
         def add_child(child)
           @children << child
         end
 
         def api_path
-          "#{parent.api_path}/#{name}/_latest"
+          "#{parent.api_path}/#{cookbook_name}/#{version || "_latest"}"
         end
 
         def child(name)
@@ -68,11 +84,8 @@ class Chef
 
         def can_have_child?(name, is_dir)
           # A cookbook's root may not have directories unless they are segment directories
-          if is_dir
-            return name != 'root_files' &&
-                   COOKBOOK_SEGMENT_INFO.keys.any? { |segment| segment.to_s == name }
-          end
-          true
+          return name != 'root_files' && COOKBOOK_SEGMENT_INFO.keys.include?(name.to_sym) if is_dir
+          return true
         end
 
         def children
@@ -100,6 +113,7 @@ class Chef
                 container.add_child(CookbookFile.new(parts[parts.length-1], container, segment_file))
               end
             end
+            @children = @children.sort_by { |c| c.name }
           end
           @children
         end
@@ -108,17 +122,32 @@ class Chef
           exists?
         end
 
-        def read
-          # This will only be called if dir? is false, which means exists? is false.
-          raise Chef::ChefFS::FileSystem::NotFoundError, path_for_printing
+        def delete(recurse)
+          if recurse
+            begin
+              rest.delete_rest(api_path)
+            rescue Timeout::Error => e
+              raise Chef::ChefFS::FileSystem::OperationFailedError.new(:delete, self, e), "Timeout deleting: #{e}"
+            rescue Net::HTTPServerException
+              if $!.response.code == "404"
+                raise Chef::ChefFS::FileSystem::NotFoundError.new(self, $!)
+              else
+                raise Chef::ChefFS::FileSystem::OperationFailedError.new(:delete, self, e), "HTTP error deleting: #{e}"
+              end
+            end
+          else
+            raise NotFoundError.new(self) if !exists?
+            raise MustDeleteRecursivelyError.new(self), "#{path_for_printing} must be deleted recursively"
+          end
         end
 
+        # In versioned cookbook mode, actually check if the version exists
+        # Probably want to cache this.
         def exists?
-          if !@versions
-            child = parent.children.select { |child| child.name == name }.first
-            @versions = child.versions if child
+          if @exists.nil?
+            @exists = parent.children.any? { |child| child.name == name }
           end
-          !!@versions
+          @exists
         end
 
         def compare_to(other)
@@ -126,14 +155,16 @@ class Chef
             return [ !exists?, nil, nil ]
           end
           are_same = true
-          Chef::ChefFS::CommandLine::diff_entries(self, other, nil, :name_only) do
-            are_same = false
+          Chef::ChefFS::CommandLine::diff_entries(self, other, nil, :name_only).each do |type, old_entry, new_entry|
+            if [ :directory_to_file, :file_to_directory, :deleted, :added, :modified ].include?(type)
+              are_same = false
+            end
           end
           [ are_same, nil, nil ]
         end
 
-        def copy_from(other)
-          parent.upload_cookbook_from(other)
+        def copy_from(other, options = {})
+          parent.upload_cookbook_from(other, options)
         end
 
         def rest
@@ -147,7 +178,7 @@ class Chef
 
           # The negative (not found) response is cached
           if @could_not_get_chef_object
-            raise Chef::ChefFS::FileSystem::NotFoundError.new(@could_not_get_chef_object), "#{path_for_printing} not found"
+            raise Chef::ChefFS::FileSystem::NotFoundError.new(self, @could_not_get_chef_object)
           end
 
           begin
@@ -159,26 +190,30 @@ class Chef
             old_retry_count = Chef::Config[:http_retry_count]
             begin
               Chef::Config[:http_retry_count] = 0
-              @chef_object ||= rest.get_rest(api_path)
+              @chef_object ||= Chef::CookbookVersion.json_create(Chef::ChefFS::RawRequest.raw_json(rest, api_path))
             ensure
               Chef::Config[:http_retry_count] = old_retry_count
             end
-          rescue Net::HTTPServerException
-            if $!.response.code == "404"
-              @could_not_get_chef_object = $!
-              raise Chef::ChefFS::FileSystem::NotFoundError.new(@could_not_get_chef_object), "#{path_for_printing} not found"
+
+          rescue Timeout::Error => e
+            raise Chef::ChefFS::FileSystem::OperationFailedError.new(:read, self, e), "Timeout reading: #{e}"
+
+          rescue Net::HTTPServerException => e
+            if e.response.code == "404"
+              @could_not_get_chef_object = e
+              raise Chef::ChefFS::FileSystem::NotFoundError.new(self, @could_not_get_chef_object)
             else
-              raise
+              raise Chef::ChefFS::FileSystem::OperationFailedError.new(:read, self, e), "HTTP error reading: #{e}"
             end
 
           # Chef bug http://tickets.opscode.com/browse/CHEF-3066 ... instead of 404 we get 500 right now.
           # Remove this when that bug is fixed.
-          rescue Net::HTTPFatalError
-            if $!.response.code == "500"
-              @could_not_get_chef_object = $!
-              raise Chef::ChefFS::FileSystem::NotFoundError.new(@could_not_get_chef_object), "#{path_for_printing} not found"
+          rescue Net::HTTPFatalError => e
+            if e.response.code == "500"
+              @could_not_get_chef_object = e
+              raise Chef::ChefFS::FileSystem::NotFoundError.new(self, @could_not_get_chef_object)
             else
-              raise
+              raise Chef::ChefFS::FileSystem::OperationFailedError.new(:read, self, e), "HTTP error reading: #{e}"
             end
           end
         end
diff --git a/lib/chef/chef_fs/file_system/cookbook_file.rb b/lib/chef/chef_fs/file_system/cookbook_file.rb
index baa71f5d9e..e05c4aa614 100644
--- a/lib/chef/chef_fs/file_system/cookbook_file.rb
+++ b/lib/chef/chef_fs/file_system/cookbook_file.rb
@@ -38,10 +38,21 @@ class Chef
           old_sign_on_redirect = rest.sign_on_redirect
           rest.sign_on_redirect = false
           begin
-            rest.get_rest(file[:url])
+            tmpfile = rest.get_rest(file[:url], true)
+          rescue Timeout::Error => e
+            raise Chef::ChefFS::FileSystem::OperationFailedError.new(:read, self, e), "Timeout reading #{file[:url]}: #{e}"
+          rescue Net::HTTPServerException => e
+            raise Chef::ChefFS::FileSystem::OperationFailedError.new(:read, self, e), "#{e.message} retrieving #{file[:url]}"
           ensure
             rest.sign_on_redirect = old_sign_on_redirect
           end
+
+          begin
+            tmpfile.open
+            tmpfile.read
+          ensure
+            tmpfile.close!
+          end
         end
 
         def rest
@@ -66,11 +77,7 @@ class Chef
         private
 
         def calc_checksum(value)
-          begin
-            Digest::MD5.hexdigest(value)
-          rescue Chef::ChefFS::FileSystem::NotFoundError
-            nil
-          end
+          Digest::MD5.hexdigest(value)
         end
       end
     end
diff --git a/lib/chef/chef_fs/file_system/cookbook_frozen_error.rb b/lib/chef/chef_fs/file_system/cookbook_frozen_error.rb
new file mode 100644
index 0000000000..705673384d
--- /dev/null
+++ b/lib/chef/chef_fs/file_system/cookbook_frozen_error.rb
@@ -0,0 +1,31 @@
+#
+# Author:: John Keiser (<jkeiser@opscode.com>)
+# Copyright:: Copyright (c) 2012 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/chef_fs/file_system/already_exists_error'
+
+class Chef
+  module ChefFS
+    module FileSystem
+      class CookbookFrozenError < AlreadyExistsError
+        def initialize(operation, entry, cause = nil)
+          super(operation, entry, cause)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/chef/chef_fs/file_system/cookbooks_acl_dir.rb b/lib/chef/chef_fs/file_system/cookbooks_acl_dir.rb
new file mode 100644
index 0000000000..d6246f1e60
--- /dev/null
+++ b/lib/chef/chef_fs/file_system/cookbooks_acl_dir.rb
@@ -0,0 +1,41 @@
+#
+# Author:: John Keiser (<jkeiser@opscode.com>)
+# Copyright:: Copyright (c) 2013 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/chef_fs/file_system/acl_dir'
+require 'chef/chef_fs/file_system/acl_entry'
+
+class Chef
+  module ChefFS
+    module FileSystem
+      class CookbooksAclDir < AclDir
+        # If versioned_cookbooks is on, the list of cookbooks will have versions
+        # in them.  But all versions of a cookbook have the same acl, so even if
+        # we have cookbooks/apache2-1.0.0 and cookbooks/apache2-1.1.2, we will
+        # only have one acl: acls/cookbooks/apache2.json.  Thus, the list of
+        # children of acls/cookbooks is a unique list of cookbook *names*.
+        def children
+          if @children.nil?
+            names = parent.parent.child(name).children.map { |child| "#{child.cookbook_name}.json" }
+            @children = names.uniq.map { |name| AclEntry.new(name, self, true) }
+          end
+          @children
+        end
+      end
+    end
+  end
+end
diff --git a/lib/chef/chef_fs/file_system/cookbooks_dir.rb b/lib/chef/chef_fs/file_system/cookbooks_dir.rb
index 9249b42aaa..193788dc23 100644
--- a/lib/chef/chef_fs/file_system/cookbooks_dir.rb
+++ b/lib/chef/chef_fs/file_system/cookbooks_dir.rb
@@ -18,6 +18,11 @@
 
 require 'chef/chef_fs/file_system/rest_list_dir'
 require 'chef/chef_fs/file_system/cookbook_dir'
+require 'chef/chef_fs/raw_request'
+require 'chef/chef_fs/file_system/operation_failed_error'
+require 'chef/chef_fs/file_system/cookbook_frozen_error'
+
+require 'tmpdir'
 
 class Chef
   module ChefFS
@@ -28,39 +33,114 @@ class Chef
         end
 
         def child(name)
-          result = @children.select { |child| child.name == name }.first if @children
-          result || CookbookDir.new(name, self)
+          if @children
+            result = self.children.select { |child| child.name == name }.first
+            if result
+              result
+            else
+              NonexistentFSObject.new(name, self)
+            end
+          else
+            CookbookDir.new(name, self)
+          end
         end
 
         def children
-          @children ||= rest.get_rest(api_path).map { |key, value| CookbookDir.new(key, self, value) }
+          @children ||= begin
+            if Chef::Config[:versioned_cookbooks]
+              result = []
+              Chef::ChefFS::RawRequest.raw_json(rest, "#{api_path}/?num_versions=all").each_pair do |cookbook_name, cookbooks|
+                cookbooks['versions'].each do |cookbook_version|
+                  result << CookbookDir.new("#{cookbook_name}-#{cookbook_version['version']}", self, :exists => true)
+                end
+              end
+            else
+              result = Chef::ChefFS::RawRequest.raw_json(rest, api_path).keys.map { |cookbook_name| CookbookDir.new(cookbook_name, self, :exists => true) }
+            end
+            result.sort_by(&:name)
+          end
         end
 
-        def create_child_from(other)
-          upload_cookbook_from(other)
+        def create_child_from(other, options = {})
+          upload_cookbook_from(other, options)
         end
 
-        def upload_cookbook_from(other)
-          other_cookbook_version = other.chef_object
-          # TODO this only works on the file system.  And it can't be broken into
-          # pieces.
-          begin
-            uploader = Chef::CookbookUploader.new(other_cookbook_version, other.parent.file_path)
-            uploader.upload_cookbooks
-          rescue Net::HTTPServerException => e
-            case e.response.code
-            when "409"
-              ui.error "Version #{other_cookbook_version.version} of cookbook #{other_cookbook_version.name} is frozen. Use --force to override."
-              Chef::Log.debug(e)
-              raise Exceptions::CookbookFrozen
-            else
-              raise
+        def upload_cookbook_from(other, options = {})
+          Chef::Config[:versioned_cookbooks] ? upload_versioned_cookbook(other, options) : upload_unversioned_cookbook(other, options)
+        rescue Timeout::Error => e
+          raise Chef::ChefFS::FileSystem::OperationFailedError.new(:write, self, e), "Timeout writing: #{e}"
+        rescue Net::HTTPServerException => e
+          case e.response.code
+          when "409"
+            raise Chef::ChefFS::FileSystem::CookbookFrozenError.new(:write, self, e), "Cookbook #{other.name} is frozen"
+          else
+            raise Chef::ChefFS::FileSystem::OperationFailedError.new(:write, self, e), "HTTP error writing: #{e}"
+          end
+        rescue Chef::Exceptions::CookbookFrozen => e
+          raise Chef::ChefFS::FileSystem::CookbookFrozenError.new(:write, self, e), "Cookbook #{other.name} is frozen"
+        end
+
+        # Knife currently does not understand versioned cookbooks
+        # Cookbook Version uploader also requires a lot of refactoring
+        # to make this work. So instead, we make a temporary cookbook
+        # symlinking back to real cookbook, and upload the proxy.
+        def upload_versioned_cookbook(other, options)
+          cookbook_name = Chef::ChefFS::FileSystem::ChefRepositoryFileSystemEntry.canonical_cookbook_name(other.name)
+
+          Dir.mktmpdir do |temp_cookbooks_path|
+            proxy_cookbook_path = "#{temp_cookbooks_path}/#{cookbook_name}"
+
+            # Make a symlink
+            File.symlink other.file_path, proxy_cookbook_path
+
+            # Instantiate a proxy loader using the temporary symlink
+            proxy_loader = Chef::Cookbook::CookbookVersionLoader.new(proxy_cookbook_path, other.parent.chefignore)
+            proxy_loader.load_cookbooks
+
+            cookbook_to_upload = proxy_loader.cookbook_version
+            cookbook_to_upload.freeze_version if options[:freeze]
+
+            # Instantiate a new uploader based on the proxy loader
+            uploader = Chef::CookbookUploader.new(cookbook_to_upload, proxy_cookbook_path, :force => options[:force], :rest => rest)
+
+            with_actual_cookbooks_dir(temp_cookbooks_path) do
+              upload_cookbook!(uploader)
             end
           end
         end
 
+        def upload_unversioned_cookbook(other, options)
+          cookbook_to_upload = other.chef_object
+          cookbook_to_upload.freeze_version if options[:freeze]
+          uploader = Chef::CookbookUploader.new(cookbook_to_upload, other.parent.file_path, :force => options[:force], :rest => rest)
+
+          with_actual_cookbooks_dir(other.parent.file_path) do
+            upload_cookbook!(uploader)
+          end
+        end
+
+        # Work around the fact that CookbookUploader doesn't understand chef_repo_path (yet)
+        def with_actual_cookbooks_dir(actual_cookbook_path)
+          old_cookbook_path = Chef::Config.cookbook_path
+          Chef::Config.cookbook_path = actual_cookbook_path if !Chef::Config.cookbook_path
+
+          yield
+        ensure
+          Chef::Config.cookbook_path = old_cookbook_path
+        end
+
+        def upload_cookbook!(uploader, options = {})
+          if uploader.respond_to?(:upload_cookbook)
+            uploader.upload_cookbook
+          else
+            uploader.upload_cookbooks
+          end
+        end
+
         def can_have_child?(name, is_dir)
-          is_dir
+          return false if !is_dir
+          return false if Chef::Config[:versioned_cookbooks] && name !~ Chef::ChefFS::FileSystem::CookbookDir::VALID_VERSIONED_COOKBOOK_NAME
+          return true
         end
       end
     end
diff --git a/lib/chef/chef_fs/file_system/data_bag_dir.rb b/lib/chef/chef_fs/file_system/data_bag_dir.rb
index 41fb5dfc63..3814b94fac 100644
--- a/lib/chef/chef_fs/file_system/data_bag_dir.rb
+++ b/lib/chef/chef_fs/file_system/data_bag_dir.rb
@@ -17,16 +17,16 @@
 #
 
 require 'chef/chef_fs/file_system/rest_list_dir'
-require 'chef/chef_fs/file_system/data_bag_item'
 require 'chef/chef_fs/file_system/not_found_error'
 require 'chef/chef_fs/file_system/must_delete_recursively_error'
+require 'chef/chef_fs/data_handler/data_bag_item_data_handler'
 
 class Chef
   module ChefFS
     module FileSystem
       class DataBagDir < RestListDir
         def initialize(name, parent, exists = nil)
-          super(name, parent)
+          super(name, parent, nil, Chef::ChefFS::DataHandler::DataBagItemDataHandler.new)
           @exists = nil
         end
 
@@ -36,7 +36,7 @@ class Chef
 
         def read
           # This will only be called if dir? is false, which means exists? is false.
-          raise Chef::ChefFS::FileSystem::NotFoundError, "#{path_for_printing} not found"
+          raise Chef::ChefFS::FileSystem::NotFoundError.new(self)
         end
 
         def exists?
@@ -46,29 +46,20 @@ class Chef
           @exists
         end
 
-        def create_child(name, file_contents)
-          json = Chef::JSONCompat.from_json(file_contents).to_hash
-          id = name[0,name.length-5]
-          if json.include?('id') && json['id'] != id
-            raise "ID in #{path_for_printing}/#{name} must be '#{id}' (is '#{json['id']}')"
-          end
-          rest.post_rest(api_path, json)
-          _make_child_entry(name, true)
-        end
-
-        def _make_child_entry(name, exists = nil)
-          DataBagItem.new(name, self, exists)
-        end
-
         def delete(recurse)
           if !recurse
-            raise Chef::ChefFS::FileSystem::MustDeleteRecursivelyError.new, "#{path_for_printing} must be deleted recursively"
+            raise NotFoundError.new(self) if !exists?
+            raise MustDeleteRecursivelyError.new(self), "#{path_for_printing} must be deleted recursively"
           end
           begin
             rest.delete_rest(api_path)
-          rescue Net::HTTPServerException
-            if $!.response.code == "404"
-              raise Chef::ChefFS::FileSystem::NotFoundError.new($!), "#{path_for_printing} not found"
+          rescue Timeout::Error => e
+            raise Chef::ChefFS::FileSystem::OperationFailedError.new(:delete, self, e), "Timeout deleting: #{e}"
+          rescue Net::HTTPServerException => e
+            if e.response.code == "404"
+              raise Chef::ChefFS::FileSystem::NotFoundError.new(self, e)
+            else
+              raise Chef::ChefFS::FileSystem::OperationFailedError.new(:delete, self, e), "HTTP error deleting: #{e}"
             end
           end
         end
diff --git a/lib/chef/chef_fs/file_system/data_bag_item.rb b/lib/chef/chef_fs/file_system/data_bag_item.rb
deleted file mode 100644
index 2f6eb15232..0000000000
--- a/lib/chef/chef_fs/file_system/data_bag_item.rb
+++ /dev/null
@@ -1,59 +0,0 @@
-#
-# Author:: John Keiser (<jkeiser@opscode.com>)
-# Copyright:: Copyright (c) 2012 Opscode, Inc.
-# License:: Apache License, Version 2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-require 'chef/chef_fs/file_system/rest_list_entry'
-
-class Chef
-  module ChefFS
-    module FileSystem
-      class DataBagItem < RestListEntry
-        def initialize(name, parent, exists = nil)
-          super(name, parent, exists)
-        end
-
-        def write(file_contents)
-          # Write is just a little tiny bit different for data bags:
-          # you set raw_data in the JSON instead of putting the items
-          # in the top level.
-          json = Chef::JSONCompat.from_json(file_contents).to_hash
-          id = name[0,name.length-5]  # Strip off the .json from the end
-          if json['id'] != id
-            raise "Id in #{path_for_printing}/#{name} must be '#{id}' (is '#{json['id']}')"
-          end
-          begin
-            data_bag = parent.name
-            json = {
-              "name" => "data_bag_item_#{data_bag}_#{id}",
-              "json_class" => "Chef::DataBagItem",
-              "chef_type" => "data_bag_item",
-              "data_bag" => data_bag,
-              "raw_data" => json
-            }
-            rest.put_rest(api_path, json)
-          rescue Net::HTTPServerException
-            if $!.response.code == "404"
-              raise Chef::ChefFS::FileSystem::NotFoundError.new($!), "#{path_for_printing} not found"
-            else
-              raise
-            end
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/chef/chef_fs/file_system/data_bags_dir.rb b/lib/chef/chef_fs/file_system/data_bags_dir.rb
index 6eca990545..46c3c21cf6 100644
--- a/lib/chef/chef_fs/file_system/data_bags_dir.rb
+++ b/lib/chef/chef_fs/file_system/data_bags_dir.rb
@@ -34,14 +34,16 @@ class Chef
 
         def children
           begin
-            @children ||= rest.get_rest(api_path).keys.map do |entry|
+            @children ||= Chef::ChefFS::RawRequest.raw_json(rest, api_path).keys.sort.map do |entry|
               DataBagDir.new(entry, self, true)
             end
-          rescue Net::HTTPServerException
-            if $!.response.code == "404"
-              raise Chef::ChefFS::FileSystem::NotFoundError.new($!), "#{path_for_printing} not found"
+          rescue Timeout::Error => e
+            raise Chef::ChefFS::FileSystem::OperationFailedError.new(:children, self, e), "Timeout getting children: #{e}"
+          rescue Net::HTTPServerException => e
+            if e.response.code == "404"
+              raise Chef::ChefFS::FileSystem::NotFoundError.new(self, e)
             else
-              raise
+              raise Chef::ChefFS::FileSystem::OperationFailedError.new(:children, self, e), "HTTP error getting children: #{e}"
             end
           end
         end
@@ -53,9 +55,13 @@ class Chef
         def create_child(name, file_contents)
           begin
             rest.post_rest(api_path, { 'name' => name })
-          rescue Net::HTTPServerException
-            if $!.response.code != "409"
-              raise
+          rescue Timeout::Error => e
+            raise Chef::ChefFS::FileSystem::OperationFailedError.new(:create_child, self, e), "Timeout creating child '#{name}': #{e}"
+          rescue Net::HTTPServerException => e
+            if e.response.code == "409"
+              raise Chef::ChefFS::FileSystem::AlreadyExistsError.new(:create_child, self, e), "Cannot create #{name} under #{path}: already exists"
+            else
+              raise Chef::ChefFS::FileSystem::OperationFailedError.new(:create_child, self, e), "HTTP error creating child '#{name}': #{e}"
             end
           end
           DataBagDir.new(name, self, true)
diff --git a/lib/chef/chef_fs/file_system/default_environment_cannot_be_modified_error.rb b/lib/chef/chef_fs/file_system/default_environment_cannot_be_modified_error.rb
new file mode 100644
index 0000000000..8ca3b917ca
--- /dev/null
+++ b/lib/chef/chef_fs/file_system/default_environment_cannot_be_modified_error.rb
@@ -0,0 +1,36 @@
+#
+# Author:: John Keiser (<jkeiser@opscode.com>)
+# Copyright:: Copyright (c) 2012 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/chef_fs/file_system/operation_not_allowed_error'
+
+class Chef
+  module ChefFS
+    module FileSystem
+      class DefaultEnvironmentCannotBeModifiedError < OperationNotAllowedError
+        def initialize(operation, entry, cause = nil)
+          super(operation, entry, cause)
+        end
+
+        def reason
+          result = super
+          result + " (default environment cannot be modified)"
+        end
+      end
+    end
+  end
+end
diff --git a/lib/chef/chef_fs/file_system/environments_dir.rb b/lib/chef/chef_fs/file_system/environments_dir.rb
new file mode 100644
index 0000000000..559dd6af86
--- /dev/null
+++ b/lib/chef/chef_fs/file_system/environments_dir.rb
@@ -0,0 +1,60 @@
+#
+# Author:: John Keiser (<jkeiser@opscode.com>)
+# Copyright:: Copyright (c) 2012 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/chef_fs/file_system/base_fs_dir'
+require 'chef/chef_fs/file_system/rest_list_entry'
+require 'chef/chef_fs/file_system/not_found_error'
+require 'chef/chef_fs/file_system/default_environment_cannot_be_modified_error'
+require 'chef/chef_fs/data_handler/environment_data_handler'
+
+class Chef
+  module ChefFS
+    module FileSystem
+      class EnvironmentsDir < RestListDir
+        def initialize(parent)
+          super("environments", parent, nil, Chef::ChefFS::DataHandler::EnvironmentDataHandler.new)
+        end
+
+        def _make_child_entry(name, exists = nil)
+          if name == '_default.json'
+            DefaultEnvironmentEntry.new(name, self, exists)
+          else
+            super
+          end
+        end
+
+        class DefaultEnvironmentEntry < RestListEntry
+          def initialize(name, parent, exists = nil)
+            super(name, parent)
+            @exists = exists
+          end
+
+          def delete(recurse)
+            raise NotFoundError.new(self) if !exists?
+            raise DefaultEnvironmentCannotBeModifiedError.new(:delete, self), "#{path_for_printing} cannot be deleted."
+          end
+
+          def write(file_contents)
+            raise NotFoundError.new(self) if !exists?
+            raise DefaultEnvironmentCannotBeModifiedError.new(:write, self), "#{path_for_printing} cannot be updated."
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/chef/chef_fs/file_system/file_system_entry.rb b/lib/chef/chef_fs/file_system/file_system_entry.rb
index a86e0cb82a..82c52deae8 100644
--- a/lib/chef/chef_fs/file_system/file_system_entry.rb
+++ b/lib/chef/chef_fs/file_system/file_system_entry.rb
@@ -19,6 +19,7 @@
 require 'chef/chef_fs/file_system/base_fs_dir'
 require 'chef/chef_fs/file_system/rest_list_dir'
 require 'chef/chef_fs/file_system/not_found_error'
+require 'chef/chef_fs/file_system/must_delete_recursively_error'
 require 'chef/chef_fs/path_utils'
 require 'fileutils'
 
@@ -34,14 +35,14 @@ class Chef
         attr_reader :file_path
 
         def path_for_printing
-          Chef::ChefFS::PathUtils::relative_to(file_path, File.expand_path(Dir.pwd))
+          file_path
         end
 
         def children
           begin
-            @children ||= Dir.entries(file_path).select { |entry| entry != '.' && entry != '..' }.map { |entry| FileSystemEntry.new(entry, self) }
+            @children ||= Dir.entries(file_path).sort.select { |entry| entry != '.' && entry != '..' }.map { |entry| FileSystemEntry.new(entry, self) }
           rescue Errno::ENOENT
-            raise Chef::ChefFS::FileSystem::NotFoundError.new($!), "#{file_path} not found"
+            raise Chef::ChefFS::FileSystem::NotFoundError.new(self, $!)
           end
         end
 
@@ -61,11 +62,10 @@ class Chef
 
         def delete(recurse)
           if dir?
-            if recurse
-              FileUtils.rm_rf(file_path)
-            else
-              File.rmdir(file_path)
+            if !recurse
+              raise MustDeleteRecursivelyError.new(self, $!)
             end
+            FileUtils.rm_rf(file_path)
           else
             File.delete(file_path)
           end
@@ -75,7 +75,7 @@ class Chef
           begin
             File.open(file_path, "rb") {|f| f.read}
           rescue Errno::ENOENT
-            raise Chef::ChefFS::FileSystem::NotFoundError.new($!), "#{file_path} not found"
+            raise Chef::ChefFS::FileSystem::NotFoundError.new(self, $!)
           end
         end
 
diff --git a/lib/chef/chef_fs/file_system/file_system_error.rb b/lib/chef/chef_fs/file_system/file_system_error.rb
index a461221108..80aff35893 100644
--- a/lib/chef/chef_fs/file_system/file_system_error.rb
+++ b/lib/chef/chef_fs/file_system/file_system_error.rb
@@ -20,10 +20,12 @@ class Chef
   module ChefFS
     module FileSystem
       class FileSystemError < StandardError
-        def initialize(cause = nil)
+        def initialize(entry, cause = nil)
+          @entry = entry
           @cause = cause
         end
 
+        attr_reader :entry
         attr_reader :cause
       end
     end
diff --git a/lib/chef/chef_fs/file_system/memory_dir.rb b/lib/chef/chef_fs/file_system/memory_dir.rb
new file mode 100644
index 0000000000..a7eda3c654
--- /dev/null
+++ b/lib/chef/chef_fs/file_system/memory_dir.rb
@@ -0,0 +1,52 @@
+require 'chef/chef_fs/file_system/base_fs_dir'
+require 'chef/chef_fs/file_system/nonexistent_fs_object'
+require 'chef/chef_fs/file_system/memory_file'
+
+class Chef
+  module ChefFS
+    module FileSystem
+      class MemoryDir < Chef::ChefFS::FileSystem::BaseFSDir
+        def initialize(name, parent)
+          super(name, parent)
+          @children = []
+        end
+
+        attr_reader :children
+
+        def child(name)
+          @children.select { |child| child.name == name }.first || Chef::ChefFS::FileSystem::NonexistentFSObject.new(name, self)
+        end
+
+        def add_child(child)
+          @children.push(child)
+        end
+
+        def can_have_child?(name, is_dir)
+          root.cannot_be_in_regex ? (name !~ root.cannot_be_in_regex) : true
+        end
+
+        def add_file(path, value)
+          path_parts = path.split('/')
+          dir = add_dir(path_parts[0..-2].join('/'))
+          file = MemoryFile.new(path_parts[-1], dir, value)
+          dir.add_child(file)
+          file
+        end
+
+        def add_dir(path)
+          path_parts = path.split('/')
+          dir = self
+          path_parts.each do |path_part|
+            subdir = dir.child(path_part)
+            if !subdir.exists?
+              subdir = MemoryDir.new(path_part, dir)
+              dir.add_child(subdir)
+            end
+            dir = subdir
+          end
+          dir
+        end
+      end
+    end
+  end
+end
diff --git a/lib/chef/chef_fs/file_system/memory_file.rb b/lib/chef/chef_fs/file_system/memory_file.rb
new file mode 100644
index 0000000000..0c44e703f1
--- /dev/null
+++ b/lib/chef/chef_fs/file_system/memory_file.rb
@@ -0,0 +1,17 @@
+require 'chef/chef_fs/file_system/base_fs_object'
+
+class Chef
+  module ChefFS
+    module FileSystem
+      class MemoryFile < Chef::ChefFS::FileSystem::BaseFSObject
+        def initialize(name, parent, value)
+          super(name, parent)
+          @value = value
+        end
+        def read
+          return @value
+        end
+      end
+    end
+  end
+end
diff --git a/lib/chef/chef_fs/file_system/memory_root.rb b/lib/chef/chef_fs/file_system/memory_root.rb
new file mode 100644
index 0000000000..4a83830946
--- /dev/null
+++ b/lib/chef/chef_fs/file_system/memory_root.rb
@@ -0,0 +1,21 @@
+require 'chef/chef_fs/file_system/memory_dir'
+
+class Chef
+  module ChefFS
+    module FileSystem
+      class MemoryRoot < MemoryDir
+        def initialize(pretty_name, cannot_be_in_regex = nil)
+          super('', nil)
+          @pretty_name = pretty_name
+          @cannot_be_in_regex = cannot_be_in_regex
+        end
+
+        attr_reader :cannot_be_in_regex
+
+        def path_for_printing
+          @pretty_name
+        end
+      end
+    end
+  end
+end
diff --git a/lib/chef/chef_fs/file_system/multiplexed_dir.rb b/lib/chef/chef_fs/file_system/multiplexed_dir.rb
new file mode 100644
index 0000000000..a7a901e304
--- /dev/null
+++ b/lib/chef/chef_fs/file_system/multiplexed_dir.rb
@@ -0,0 +1,48 @@
+require 'chef/chef_fs/file_system/base_fs_object'
+require 'chef/chef_fs/file_system/nonexistent_fs_object'
+
+class Chef
+  module ChefFS
+    module FileSystem
+      class MultiplexedDir < BaseFSDir
+        def initialize(*multiplexed_dirs)
+          @multiplexed_dirs = multiplexed_dirs.flatten
+          super(@multiplexed_dirs[0].name, @multiplexed_dirs[0].parent)
+        end
+
+        attr_reader :multiplexed_dirs
+
+        def write_dir
+          multiplexed_dirs[0]
+        end
+
+        def children
+          @children ||= begin
+            result = []
+            seen = {}
+            # If multiple things have the same name, the first one wins.
+            multiplexed_dirs.each do |dir|
+              dir.children.each do |child|
+                if seen[child.name]
+                  Chef::Log.warn("Child with name '#{child.name}' found in multiple directories: #{seen[child.name].path_for_printing} and #{child.path_for_printing}")
+                else
+                  result << child
+                  seen[child.name] = child
+                end
+              end
+            end
+            result
+          end
+        end
+
+        def can_have_child?(name, is_dir)
+          write_dir.can_have_child?(name, is_dir)
+        end
+
+        def create_child(name, file_contents = nil)
+          write_dir.create_child(name, file_contents)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/chef/chef_fs/file_system/must_delete_recursively_error.rb b/lib/chef/chef_fs/file_system/must_delete_recursively_error.rb
index d247a5b4ed..bfa8ba28ce 100644
--- a/lib/chef/chef_fs/file_system/must_delete_recursively_error.rb
+++ b/lib/chef/chef_fs/file_system/must_delete_recursively_error.rb
@@ -22,8 +22,8 @@ class Chef
   module ChefFS
     module FileSystem
       class MustDeleteRecursivelyError < FileSystemError
-        def initialize(cause = nil)
-          super(cause)
+        def initialize(entry, cause = nil)
+          super(entry, cause)
         end
       end
     end
diff --git a/lib/chef/chef_fs/file_system/nodes_dir.rb b/lib/chef/chef_fs/file_system/nodes_dir.rb
index 4dfbf6d850..82683e81ac 100644
--- a/lib/chef/chef_fs/file_system/nodes_dir.rb
+++ b/lib/chef/chef_fs/file_system/nodes_dir.rb
@@ -19,28 +19,36 @@
 require 'chef/chef_fs/file_system/base_fs_dir'
 require 'chef/chef_fs/file_system/rest_list_entry'
 require 'chef/chef_fs/file_system/not_found_error'
+require 'chef/chef_fs/data_handler/node_data_handler'
 
 class Chef
   module ChefFS
     module FileSystem
       class NodesDir < RestListDir
         def initialize(parent)
-          super("nodes", parent)
+          super("nodes", parent, nil, Chef::ChefFS::DataHandler::NodeDataHandler.new)
         end
 
-        # Override children to respond to environment
+        # Identical to RestListDir.children, except supports environments
         def children
-          @children ||= begin
-            env_api_path = environment ? "environments/#{environment}/#{api_path}" : api_path
-            rest.get_rest(env_api_path).keys.map { |key| RestListEntry.new("#{key}.json", self, true) }
-          rescue Net::HTTPServerException
+          begin
+            @children ||= Chef::ChefFS::RawRequest.raw_json(rest, env_api_path).keys.sort.map do |key|
+              _make_child_entry("#{key}.json", true)
+            end
+          rescue Timeout::Error => e
+            raise Chef::ChefFS::FileSystem::OperationFailedError.new(:children, self, e), "Timeout retrieving children: #{e}"
+          rescue Net::HTTPServerException => e
             if $!.response.code == "404"
-              raise Chef::ChefFS::FileSystem::NotFoundError.new($!), "#{path_for_printing} not found"
+              raise Chef::ChefFS::FileSystem::NotFoundError.new(self, $!)
             else
-              raise
+              raise Chef::ChefFS::FileSystem::OperationFailedError.new(:children, self, e), "HTTP error retrieving children: #{e}"
             end
           end
-      end
+        end
+
+        def env_api_path
+          environment ? "environments/#{environment}/#{api_path}" : api_path
+        end
       end
     end
   end
diff --git a/lib/chef/chef_fs/file_system/nonexistent_fs_object.rb b/lib/chef/chef_fs/file_system/nonexistent_fs_object.rb
index dc82e83b0d..a587ab47a4 100644
--- a/lib/chef/chef_fs/file_system/nonexistent_fs_object.rb
+++ b/lib/chef/chef_fs/file_system/nonexistent_fs_object.rb
@@ -30,10 +30,6 @@ class Chef
         def exists?
           false
         end
-
-        def read
-          raise Chef::ChefFS::FileSystem::NotFoundError, "Nonexistent #{path_for_printing}"
-        end
       end
     end
   end
diff --git a/lib/chef/chef_fs/file_system/not_found_error.rb b/lib/chef/chef_fs/file_system/not_found_error.rb
index 0b608f1abf..9eab3d6131 100644
--- a/lib/chef/chef_fs/file_system/not_found_error.rb
+++ b/lib/chef/chef_fs/file_system/not_found_error.rb
@@ -22,8 +22,8 @@ class Chef
   module ChefFS
     module FileSystem
       class NotFoundError < FileSystemError
-        def initialize(cause = nil)
-          super(cause)
+        def initialize(entry, cause = nil)
+          super(entry, cause)
         end
       end
     end
diff --git a/lib/chef/chef_fs/file_system/operation_failed_error.rb b/lib/chef/chef_fs/file_system/operation_failed_error.rb
new file mode 100644
index 0000000000..1af2d2dcff
--- /dev/null
+++ b/lib/chef/chef_fs/file_system/operation_failed_error.rb
@@ -0,0 +1,34 @@
+#
+# Author:: John Keiser (<jkeiser@opscode.com>)
+# Copyright:: Copyright (c) 2012 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/chef_fs/file_system/file_system_error'
+
+class Chef
+  module ChefFS
+    module FileSystem
+      class OperationFailedError < FileSystemError
+        def initialize(operation, entry, cause = nil)
+          super(entry, cause)
+          @operation = operation
+        end
+
+        attr_reader :operation
+      end
+    end
+  end
+end
diff --git a/lib/chef/chef_fs/file_system/operation_not_allowed_error.rb b/lib/chef/chef_fs/file_system/operation_not_allowed_error.rb
new file mode 100644
index 0000000000..4b4f9742a8
--- /dev/null
+++ b/lib/chef/chef_fs/file_system/operation_not_allowed_error.rb
@@ -0,0 +1,48 @@
+#
+# Author:: John Keiser (<jkeiser@opscode.com>)
+# Copyright:: Copyright (c) 2012 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/chef_fs/file_system/file_system_error'
+
+class Chef
+  module ChefFS
+    module FileSystem
+      class OperationNotAllowedError < FileSystemError
+        def initialize(operation, entry, cause = nil)
+          super(entry, cause)
+          @operation = operation
+        end
+
+        attr_reader :operation
+        attr_reader :entry
+
+        def reason
+          case operation
+          when :delete
+            "cannot be deleted"
+          when :write
+            "cannot be updated"
+          when :create_child
+            "cannot have a child created under it"
+          when :read
+            "cannot be read"
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/chef/chef_fs/file_system/rest_list_dir.rb b/lib/chef/chef_fs/file_system/rest_list_dir.rb
index 0e8db4d7b9..594fec8ab6 100644
--- a/lib/chef/chef_fs/file_system/rest_list_dir.rb
+++ b/lib/chef/chef_fs/file_system/rest_list_dir.rb
@@ -24,12 +24,14 @@ class Chef
   module ChefFS
     module FileSystem
       class RestListDir < BaseFSDir
-        def initialize(name, parent, api_path = nil)
+        def initialize(name, parent, api_path = nil, data_handler = nil)
           super(name, parent)
           @api_path = api_path || (parent.api_path == "" ? name : "#{parent.api_path}/#{name}")
+          @data_handler = data_handler
         end
 
         attr_reader :api_path
+        attr_reader :data_handler
 
         def child(name)
           result = @children.select { |child| child.name == name }.first if @children
@@ -43,28 +45,55 @@ class Chef
 
         def children
           begin
-            @children ||= rest.get_rest(api_path).keys.map do |key|
+            @children ||= Chef::ChefFS::RawRequest.raw_json(rest, api_path).keys.sort.map do |key|
               _make_child_entry("#{key}.json", true)
             end
-          rescue Net::HTTPServerException
+          rescue Timeout::Error => e
+            raise Chef::ChefFS::FileSystem::OperationFailedError.new(:children, self, e), "Timeout retrieving children: #{e}"
+          rescue Net::HTTPServerException => e
             if $!.response.code == "404"
-              raise Chef::ChefFS::FileSystem::NotFoundError.new($!), "#{path_for_printing} not found"
+              raise Chef::ChefFS::FileSystem::NotFoundError.new(self, $!)
             else
-              raise
+              raise Chef::ChefFS::FileSystem::OperationFailedError.new(:children, self, e), "HTTP error retrieving children: #{e}"
             end
           end
         end
 
-        # NOTE if you change this significantly, you will likely need to change
-        # DataBagDir.create_child as well.
         def create_child(name, file_contents)
-          json = Chef::JSONCompat.from_json(file_contents).to_hash
-          base_name = name[0,name.length-5]
-          if json.include?('name') && json['name'] != base_name
-            raise "Name in #{path_for_printing}/#{name} must be '#{base_name}' (is '#{json['name']}')"
+          begin
+            object = JSON.parse(file_contents, :create_additions => false)
+          rescue JSON::ParserError => e
+            raise Chef::ChefFS::FileSystem::OperationFailedError.new(:create_child, self, e), "Parse error reading JSON creating child '#{name}': #{e}"
           end
-          rest.post_rest(api_path, json)
-          _make_child_entry(name, true)
+
+          result = _make_child_entry(name, true)
+
+          if data_handler
+            object = data_handler.normalize_for_post(object, result)
+            data_handler.verify_integrity(object, result) do |error|
+              raise Chef::ChefFS::FileSystem::OperationFailedError.new(:create_child, self), "Error creating '#{name}': #{error}"
+            end
+          end
+
+          begin
+            rest.post_rest(api_path, object)
+          rescue Timeout::Error => e
+            raise Chef::ChefFS::FileSystem::OperationFailedError.new(:create_child, self, e), "Timeout creating '#{name}': #{e}"
+          rescue Net::HTTPServerException => e
+            if e.response.code == "404"
+              raise Chef::ChefFS::FileSystem::NotFoundError.new(self, e)
+            elsif $!.response.code == "409"
+              raise Chef::ChefFS::FileSystem::AlreadyExistsError.new(:create_child, self, e), "Failure creating '#{name}': #{path}/#{name} already exists"
+            else
+              raise Chef::ChefFS::FileSystem::OperationFailedError.new(:create_child, self, e), "Failure creating '#{name}': #{e.message}"
+            end
+          end
+
+          result
+        end
+
+        def org
+          parent.org
         end
 
         def environment
diff --git a/lib/chef/chef_fs/file_system/rest_list_entry.rb b/lib/chef/chef_fs/file_system/rest_list_entry.rb
index dd504ef341..6e6ad12438 100644
--- a/lib/chef/chef_fs/file_system/rest_list_entry.rb
+++ b/lib/chef/chef_fs/file_system/rest_list_entry.rb
@@ -18,6 +18,8 @@
 
 require 'chef/chef_fs/file_system/base_fs_object'
 require 'chef/chef_fs/file_system/not_found_error'
+require 'chef/chef_fs/file_system/operation_failed_error'
+require 'chef/chef_fs/raw_request'
 require 'chef/role'
 require 'chef/node'
 
@@ -30,14 +32,25 @@ class Chef
           @exists = exists
         end
 
-        def api_path
+        def data_handler
+          parent.data_handler
+        end
+
+        def api_child_name
           if name.length < 5 || name[-5,5] != ".json"
             raise "Invalid name #{path}: must end in .json"
           end
-          api_child_name = name[0,name.length-5]
+          name[0,name.length-5]
+        end
+
+        def api_path
           "#{parent.api_path}/#{api_child_name}"
         end
 
+        def org
+          parent.org
+        end
+
         def environment
           parent.environment
         end
@@ -56,45 +69,76 @@ class Chef
         def delete(recurse)
           begin
             rest.delete_rest(api_path)
-          rescue Net::HTTPServerException
-            if $!.response.code == "404"
-              raise Chef::ChefFS::FileSystem::NotFoundError.new($!), "#{path_for_printing} not found"
+          rescue Timeout::Error => e
+            raise Chef::ChefFS::FileSystem::OperationFailedError.new(:delete, self, e), "Timeout deleting: #{e}"
+          rescue Net::HTTPServerException => e
+            if e.response.code == "404"
+              raise Chef::ChefFS::FileSystem::NotFoundError.new(self, e)
             else
-              raise
+              raise Chef::ChefFS::FileSystem::OperationFailedError.new(:delete, self, e), "Timeout deleting: #{e}"
             end
           end
         end
 
         def read
-          Chef::JSONCompat.to_json_pretty(chef_object.to_hash)
+          Chef::JSONCompat.to_json_pretty(_read_hash)
         end
 
-        def chef_object
+        def _read_hash
           begin
-            # REST will inflate the Chef object using json_class
-            rest.get_rest(api_path)
-          rescue Net::HTTPServerException
+            json = Chef::ChefFS::RawRequest.raw_request(rest, api_path)
+          rescue Timeout::Error => e
+            raise Chef::ChefFS::FileSystem::OperationFailedError.new(:read, self, e), "Timeout reading: #{e}"
+          rescue Net::HTTPServerException => e
             if $!.response.code == "404"
-              raise Chef::ChefFS::FileSystem::NotFoundError.new($!), "#{path_for_printing} not found"
+              raise Chef::ChefFS::FileSystem::NotFoundError.new(self, e)
             else
-              raise
+              raise Chef::ChefFS::FileSystem::OperationFailedError.new(:read, self, e), "HTTP error reading: #{e}"
             end
           end
+          # Minimize the value (get rid of defaults) so the results don't look terrible
+          minimize_value(JSON.parse(json, :create_additions => false))
+        end
+
+        def chef_object
+          # REST will inflate the Chef object using json_class
+          data_handler.json_class.json_create(read)
+        end
+
+        def minimize_value(value)
+          data_handler.minimize(data_handler.normalize(value, self), self)
         end
 
         def compare_to(other)
+          # TODO this pair of reads can be parallelized
+
+          # Grab the other value
           begin
-            other_value = other.read
+            other_value_json = other.read
           rescue Chef::ChefFS::FileSystem::NotFoundError
             return [ nil, nil, :none ]
           end
+
+          # Grab this value
           begin
-            value = chef_object.to_hash
+            value = _read_hash
           rescue Chef::ChefFS::FileSystem::NotFoundError
-            return [ false, :none, other_value ]
+            return [ false, :none, other_value_json ]
           end
-          are_same = (value == Chef::JSONCompat.from_json(other_value, :create_additions => false))
-          [ are_same, Chef::JSONCompat.to_json_pretty(value), other_value ]
+
+          # Minimize (and normalize) both values for easy and beautiful diffs
+          value = minimize_value(value)
+          value_json = Chef::JSONCompat.to_json_pretty(value)
+          begin
+            other_value = JSON.parse(other_value_json, :create_additions => false)
+          rescue JSON::ParserError => e
+            Chef::Log.warn("Parse error reading #{other.path_for_printing} as JSON: #{e}")
+            return [ nil, value_json, other_value_json ]
+          end
+          other_value = minimize_value(other_value)
+          other_value_json = Chef::JSONCompat.to_json_pretty(other_value)
+
+          [ value == other_value, value_json, other_value_json ]
         end
 
         def rest
@@ -102,18 +146,28 @@ class Chef
         end
 
         def write(file_contents)
-          json = Chef::JSONCompat.from_json(file_contents).to_hash
-          base_name = name[0,name.length-5]
-          if json['name'] != base_name
-            raise "Name in #{path_for_printing}/#{name} must be '#{base_name}' (is '#{json['name']}')"
+          begin
+            object = JSON.parse(file_contents, :create_additions => false)
+          rescue JSON::ParserError => e
+            raise Chef::ChefFS::FileSystem::OperationFailedError.new(:write, self, e), "Parse error reading JSON: #{e}"
+          end
+
+          if data_handler
+            object = data_handler.normalize_for_put(object, self)
+            data_handler.verify_integrity(object, self) do |error|
+              raise Chef::ChefFS::FileSystem::OperationFailedError.new(:write, self), "#{error}"
+            end
           end
+
           begin
-            rest.put_rest(api_path, json)
-          rescue Net::HTTPServerException
-            if $!.response.code == "404"
-              raise Chef::ChefFS::FileSystem::NotFoundError.new($!), "#{path_for_printing} not found"
+            rest.put_rest(api_path, object)
+          rescue Timeout::Error => e
+            raise Chef::ChefFS::FileSystem::OperationFailedError.new(:write, self, e), "Timeout writing: #{e}"
+          rescue Net::HTTPServerException => e
+            if e.response.code == "404"
+              raise Chef::ChefFS::FileSystem::NotFoundError.new(self, e)
             else
-              raise
+              raise Chef::ChefFS::FileSystem::OperationFailedError.new(:write, self, e), "HTTP error writing: #{e}"
             end
           end
         end
diff --git a/lib/chef/chef_fs/knife.rb b/lib/chef/chef_fs/knife.rb
index 8a116d980e..5900c29f61 100644
--- a/lib/chef/chef_fs/knife.rb
+++ b/lib/chef/chef_fs/knife.rb
@@ -16,52 +16,74 @@
 # limitations under the License.
 #
 
-require 'chef/chef_fs/file_system/chef_server_root_dir'
-require 'chef/chef_fs/file_system/chef_repository_file_system_root_dir'
-require 'chef/chef_fs/file_pattern'
-require 'chef/chef_fs/path_utils'
-require 'chef/config'
+require 'chef/knife'
 
 class Chef
   module ChefFS
     class Knife < Chef::Knife
-      def self.common_options
+      # Workaround for CHEF-3932
+      def self.deps
+        super do
+          require 'chef/config'
+          require 'chef/chef_fs/parallelizer'
+          require 'chef/chef_fs/config'
+          require 'chef/chef_fs/file_pattern'
+          require 'chef/chef_fs/path_utils'
+          yield
+        end
+      end
+
+      def self.inherited(c)
+        super
+        # Ensure we always get to do our includes, whether subclass calls deps or not
+        c.deps do
+        end
+
         option :repo_mode,
           :long => '--repo-mode MODE',
-          :default => "default",
-          :description => "Specifies the local repository layout.  Values: default or full"
+          :description => "Specifies the local repository layout.  Values: static, everything, hosted_everything.  Default: everything/hosted_everything"
+
+        option :chef_repo_path,
+          :long => '--chef-repo-path PATH',
+          :description => 'Overrides the location of chef repo. Default is specified by chef_repo_path in the config'
+
+        option :concurrency,
+          :long => '--concurrency THREADS',
+          :description => 'Maximum number of simultaneous requests to send (default: 10)'
       end
 
-      def base_path
-        @base_path ||= begin
-          relative_to_base = Chef::ChefFS::PathUtils::relative_to(File.expand_path(Dir.pwd), chef_repo)
-          relative_to_base == '.' ? '/' : "/#{relative_to_base}"
+      def configure_chef
+        super
+        Chef::Config[:repo_mode] = config[:repo_mode] if config[:repo_mode]
+        Chef::Config[:concurrency] = config[:concurrency].to_i if config[:concurrency]
+
+        # --chef-repo-path overrides all other paths
+        if config[:chef_repo_path]
+          Chef::Config[:chef_repo_path] = config[:chef_repo_path]
+          Chef::ChefFS::Config::PATH_VARIABLES.each do |variable_name|
+            Chef::Config[variable_name.to_sym] = chef_repo_paths.map { |path| File.join(path, "#{variable_name[0..-6]}s") }
+          end
         end
+
+        @chef_fs_config = Chef::ChefFS::Config.new(Chef::Config)
+
+        Chef::ChefFS::Parallelizer.threads = (Chef::Config[:concurrency] || 10) - 1
       end
 
       def chef_fs
-        @chef_fs ||= Chef::ChefFS::FileSystem::ChefServerRootDir.new("remote", Chef::Config, config[:repo_mode])
+        @chef_fs_config.chef_fs
       end
 
-      def chef_repo
-        @chef_repo ||= File.expand_path(File.join(Chef::Config.cookbook_path, ".."))
+      def create_chef_fs
+        @chef_fs_config.create_chef_fs
       end
 
-      def format_path(path)
-        if path[0,base_path.length] == base_path
-          if path == base_path
-            return "."
-          elsif path[base_path.length] == "/"
-            return path[base_path.length + 1, path.length - base_path.length - 1]
-          elsif base_path == "/" && path[0] == "/"
-            return path[1, path.length - 1]
-          end
-        end
-        path
+      def local_fs
+        @chef_fs_config.local_fs
       end
 
-      def local_fs
-        @local_fs ||= Chef::ChefFS::FileSystem::ChefRepositoryFileSystemRootDir.new(chef_repo)
+      def create_local_fs
+        @chef_fs_config.create_local_fs
       end
 
       def pattern_args
@@ -69,9 +91,26 @@ class Chef
       end
 
       def pattern_args_from(args)
-        args.map { |arg| Chef::ChefFS::FilePattern::relative_to(base_path, arg) }.to_a
+        # TODO support absolute file paths and not just patterns?  Too much?
+        # Could be super useful in a world with multiple repo paths
+        args.map do |arg|
+          if !@chef_fs_config.base_path && !Chef::ChefFS::PathUtils.is_absolute?(arg)
+            # Check if chef repo path is specified to give a better error message
+            @chef_fs_config.require_chef_repo_path
+            ui.error("Attempt to use relative path '#{arg}' when current directory is outside the repository path")
+            exit(1)
+          end
+          Chef::ChefFS::FilePattern.relative_to(@chef_fs_config.base_path, arg)
+        end
       end
 
+      def format_path(entry)
+        @chef_fs_config.format_path(entry)
+      end
+
+      def parallelize(inputs, options = {}, &block)
+        Chef::ChefFS::Parallelizer.parallelize(inputs, options, &block)
+      end
     end
   end
 end
diff --git a/lib/chef/chef_fs/parallelizer.rb b/lib/chef/chef_fs/parallelizer.rb
new file mode 100644
index 0000000000..84f3d4d870
--- /dev/null
+++ b/lib/chef/chef_fs/parallelizer.rb
@@ -0,0 +1,129 @@
+class Chef
+  module ChefFS
+    class Parallelizer
+      @@parallelizer = nil
+      @@threads = 0
+
+      def self.threads=(value)
+        if @@threads != value
+          @@threads = value
+          @@parallelizer = nil
+        end
+      end
+
+      def self.parallelize(enumerator, options = {}, &block)
+        @@parallelizer ||= Parallelizer.new(@@threads)
+        @@parallelizer.parallelize(enumerator, options, &block)
+      end
+
+      def initialize(threads)
+        @tasks_mutex = Mutex.new
+        @tasks = []
+        @threads = []
+        1.upto(threads) do
+          @threads << Thread.new { worker_loop }
+        end
+      end
+
+      def parallelize(enumerator, options = {}, &block)
+        task = ParallelizedResults.new(enumerator, options, &block)
+        @tasks_mutex.synchronize do
+          @tasks << task
+        end
+        task
+      end
+
+      class ParallelizedResults
+        include Enumerable
+
+        def initialize(enumerator, options, &block)
+          @inputs = enumerator.to_a
+          @options = options
+          @block = block
+
+          @mutex = Mutex.new
+          @outputs = []
+          @status = []
+        end
+
+        def each
+          next_index = 0
+          while true
+            # Report any results that already exist
+            while @status.length > next_index && ([:finished, :exception].include?(@status[next_index]))
+              if @status[next_index] == :finished
+                if @options[:flatten]
+                  @outputs[next_index].each do |entry|
+                    yield entry
+                  end
+                else
+                  yield @outputs[next_index]
+                end
+              else
+                raise @outputs[next_index]
+              end
+              next_index = next_index + 1
+            end
+
+            # Pick up a result and process it, if there is one.  This ensures we
+            # move forward even if there are *zero* worker threads available.
+            if !process_input
+              # Exit if we're done.
+              if next_index >= @status.length
+                break
+              else
+                # Ruby 1.8 threading sucks.  Wait till we process more things.
+                sleep(0.05)
+              end
+            end
+          end
+        end
+
+        def process_input
+          # Grab the next one to process
+          index, input = @mutex.synchronize do
+            index = @status.length
+            if index >= @inputs.length
+              return nil
+            end
+            input = @inputs[index]
+            @status[index] = :started
+            [ index, input ]
+          end
+
+          begin
+            @outputs[index] = @block.call(input)
+            @status[index] = :finished
+          rescue Exception
+            @outputs[index] = $!
+            @status[index] = :exception
+          end
+          index
+        end
+      end
+
+      private
+
+      def worker_loop
+        while true
+          begin
+            task = @tasks[0]
+            if task
+              if !task.process_input
+                @tasks_mutex.synchronize do
+                  @tasks.delete(task)
+                end
+              end
+            else
+              # Ruby 1.8 threading sucks.  Wait a bit to see if another task comes in.
+              sleep(0.05)
+            end
+          rescue
+            puts "ERROR #{$!}"
+            puts $!.backtrace
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/chef/chef_fs/path_utils.rb b/lib/chef/chef_fs/path_utils.rb
index 67c62a7545..805b092b3a 100644
--- a/lib/chef/chef_fs/path_utils.rb
+++ b/lib/chef/chef_fs/path_utils.rb
@@ -17,6 +17,7 @@
 #
 
 require 'chef/chef_fs'
+require 'pathname'
 
 class Chef
   module ChefFS
@@ -24,13 +25,13 @@ class Chef
 
       # If you are in 'source', this is what you would have to type to reach 'dest'
       # relative_to('/a/b/c/d/e', '/a/b/x/y') == '../../c/d/e'
-      # relative_to('/a/b', '/a/b') == ''
+      # relative_to('/a/b', '/a/b') == '.'
       def self.relative_to(dest, source)
         # Skip past the common parts
         source_parts = Chef::ChefFS::PathUtils.split(source)
         dest_parts = Chef::ChefFS::PathUtils.split(dest)
         i = 0
-        until i >= source_parts.length || i >= dest_parts.length || source_parts[i] != source_parts[i]
+        until i >= source_parts.length || i >= dest_parts.length || source_parts[i] != dest_parts[i]
           i+=1
         end
         # dot-dot up from 'source' to the common ancestor, then
@@ -56,9 +57,34 @@ class Chef
       end
 
       def self.regexp_path_separator
-        Chef::ChefFS::windows? ? '[/\\]' : '/'
+        Chef::ChefFS::windows? ? '[\/\\\\]' : '/'
       end
 
+      # Given a path which may only be partly real (i.e. /x/y/z when only /x exists,
+      # or /x/y/*/blah when /x/y/z/blah exists), call File.realpath on the biggest
+      # part that actually exists.
+      #
+      # If /x is a symlink to /blarghle, and has no subdirectories, then:
+      # PathUtils.realest_path('/x/y/z') == '/blarghle/y/z'
+      # PathUtils.realest_path('/x/*/z') == '/blarghle/*/z'
+      # PathUtils.realest_path('/*/y/z') == '/*/y/z'
+      def self.realest_path(path)
+        path = Pathname.new(path)
+        begin
+          path.realpath.to_s
+        rescue Errno::ENOENT
+          dirname = path.dirname
+          if dirname
+            PathUtils.join(realest_path(dirname), path.basename.to_s)
+          else
+            path.to_s
+          end
+        end
+      end
+
+      def self.is_absolute?(path)
+        path =~ /^#{regexp_path_separator}/
+      end
     end
   end
 end
diff --git a/lib/chef/chef_fs/raw_request.rb b/lib/chef/chef_fs/raw_request.rb
new file mode 100644
index 0000000000..43907282f6
--- /dev/null
+++ b/lib/chef/chef_fs/raw_request.rb
@@ -0,0 +1,79 @@
+class Chef
+  module ChefFS
+    module RawRequest
+      def self.raw_json(chef_rest, api_path)
+        JSON.parse(raw_request(chef_rest, api_path), :create_additions => false)
+      end
+
+      def self.raw_request(chef_rest, api_path)
+        api_request(chef_rest, :GET, chef_rest.create_url(api_path), {}, false)
+      end
+
+      def self.api_request(chef_rest, method, url, headers={}, data=false)
+        json_body = data
+        #        json_body = data ? Chef::JSONCompat.to_json(data) : nil
+        # Force encoding to binary to fix SSL related EOFErrors
+        # cf. http://tickets.opscode.com/browse/CHEF-2363
+        # http://redmine.ruby-lang.org/issues/5233
+        #        json_body.force_encoding(Encoding::BINARY) if json_body.respond_to?(:force_encoding)
+        headers = build_headers(chef_rest, method, url, headers, json_body)
+
+        chef_rest.retriable_rest_request(method, url, json_body, headers) do |rest_request|
+          response = rest_request.call {|r| r.read_body}
+
+          response_body = chef_rest.decompress_body(response)
+
+          if response.kind_of?(Net::HTTPSuccess)
+            response_body
+          elsif redirect_location = redirected_to(response)
+            if [:GET, :HEAD].include?(method)
+              chef_rest.follow_redirect do
+                api_request(chef_rest, method, chef_rest.create_url(redirect_location))
+              end
+            else
+              raise Exceptions::InvalidRedirect, "#{method} request was redirected from #{url} to #{redirect_location}. Only GET and HEAD support redirects."
+            end
+          else
+            # have to decompress the body before making an exception for it. But the body could be nil.
+            response.body.replace(chef_rest.decompress_body(response)) if response.body.respond_to?(:replace)
+
+            if response['content-type'] =~ /json/
+              exception = response_body
+              msg = "HTTP Request Returned #{response.code} #{response.message}: "
+              msg << (exception["error"].respond_to?(:join) ? exception["error"].join(", ") : exception["error"].to_s)
+              Chef::Log.info(msg)
+            end
+            response.error!
+          end
+        end
+      end
+
+      private
+
+      # Copied so that it does not automatically inflate an object
+      # This is also used by knife raw_essentials
+
+      ACCEPT_ENCODING = "Accept-Encoding".freeze
+      ENCODING_GZIP_DEFLATE = "gzip;q=1.0,deflate;q=0.6,identity;q=0.3".freeze
+
+      def self.redirected_to(response)
+        return nil  unless response.kind_of?(Net::HTTPRedirection)
+        # Net::HTTPNotModified is undesired subclass of Net::HTTPRedirection so test for this
+        return nil  if response.kind_of?(Net::HTTPNotModified)
+        response['location']
+      end
+
+      def self.build_headers(chef_rest, method, url, headers={}, json_body=false, raw=false)
+        #        headers                 = @default_headers.merge(headers)
+        #headers['Accept']       = "application/json" unless raw
+        headers['Accept']       = "application/json" unless raw
+        headers["Content-Type"] = 'application/json' if json_body
+        headers['Content-Length'] = json_body.bytesize.to_s if json_body
+        headers[Chef::REST::RESTRequest::ACCEPT_ENCODING] = Chef::REST::RESTRequest::ENCODING_GZIP_DEFLATE
+        headers.merge!(chef_rest.authentication_headers(method, url, json_body)) if chef_rest.sign_requests?
+        headers.merge!(Chef::Config[:custom_http_headers]) if Chef::Config[:custom_http_headers]
+        headers
+      end
+    end
+  end
+end
diff --git a/lib/chef/knife/delete.rb b/lib/chef/knife/delete.rb
index 060e32ba99..fb26b9ea35 100644
--- a/lib/chef/knife/delete.rb
+++ b/lib/chef/knife/delete.rb
@@ -1,18 +1,30 @@
 require 'chef/chef_fs/knife'
-require 'chef/chef_fs/file_system'
 
 class Chef
   class Knife
     class Delete < Chef::ChefFS::Knife
       banner "knife delete [PATTERN1 ... PATTERNn]"
 
-      common_options
+      deps do
+        require 'chef/chef_fs/file_system'
+      end
 
       option :recurse,
+        :short => '-r',
         :long => '--[no-]recurse',
         :boolean => true,
         :default => false,
         :description => "Delete directories recursively."
+      option :both,
+        :long => '--both',
+        :boolean => true,
+        :default => false,
+        :description => "Delete both the local and remote copies."
+      option :local,
+        :long => '--local',
+        :boolean => true,
+        :default => false,
+        :description => "Delete the local copy (leave the remote copy)."
 
       def run
         if name_args.length == 0
@@ -22,16 +34,71 @@ class Chef
         end
 
         # Get the matches (recursively)
-        pattern_args.each do |pattern|
-          Chef::ChefFS::FileSystem.list(chef_fs, pattern) do |result|
-            begin
-              result.delete(config[:recurse])
-              puts "Deleted #{result.path_for_printing}"
-            rescue Chef::ChefFS::FileSystem::NotFoundError
-              STDERR.puts "#{result.path_for_printing}: No such file or directory"
+        error = false
+        if config[:local]
+          pattern_args.each do |pattern|
+            Chef::ChefFS::FileSystem.list(local_fs, pattern).each do |result|
+              if delete_result(result)
+                error = true
+              end
             end
           end
+        elsif config[:both]
+          pattern_args.each do |pattern|
+            Chef::ChefFS::FileSystem.list_pairs(pattern, chef_fs, local_fs).each do |chef_result, local_result|
+              if delete_result(chef_result, local_result)
+                error = true
+              end
+            end
+          end
+        else # Remote only
+          pattern_args.each do |pattern|
+            Chef::ChefFS::FileSystem.list(chef_fs, pattern).each do |result|
+              if delete_result(result)
+                error = true
+              end
+            end
+          end
+        end
+
+        if error
+          exit 1
+        end
+      end
+
+      def format_path_with_root(entry)
+        root = entry.root == chef_fs ? " (remote)" : " (local)"
+        "#{format_path(entry)}#{root}"
+      end
+
+      def delete_result(*results)
+        deleted_any = false
+        found_any = false
+        error = false
+        results.each do |result|
+          begin
+            result.delete(config[:recurse])
+            deleted_any = true
+            found_any = true
+          rescue Chef::ChefFS::FileSystem::NotFoundError
+            # This is not an error unless *all* of them were not found
+          rescue Chef::ChefFS::FileSystem::MustDeleteRecursivelyError => e
+            ui.error "#{format_path_with_root(e.entry)} must be deleted recursively!  Pass -r to knife delete."
+            found_any = true
+            error = true
+          rescue Chef::ChefFS::FileSystem::OperationNotAllowedError => e
+            ui.error "#{format_path_with_root(e.entry)} #{e.reason}."
+            found_any = true
+            error = true
+          end
+        end
+        if deleted_any
+          output("Deleted #{format_path(results[0])}")
+        elsif !found_any
+          ui.error "#{format_path(results[0])}: No such file or directory"
+          error = true
         end
+        error
       end
     end
   end
diff --git a/lib/chef/knife/deps.rb b/lib/chef/knife/deps.rb
new file mode 100644
index 0000000000..c4b3678ff8
--- /dev/null
+++ b/lib/chef/knife/deps.rb
@@ -0,0 +1,139 @@
+require 'chef/chef_fs/knife'
+
+class Chef
+  class Knife
+    class Deps < Chef::ChefFS::Knife
+      banner "knife deps PATTERN1 [PATTERNn]"
+
+      deps do
+        require 'chef/chef_fs/file_system'
+        require 'chef/run_list'
+      end
+
+      option :recurse,
+        :long => '--[no-]recurse',
+        :boolean => true,
+        :description => "List dependencies recursively (default: true).  Only works with --tree."
+      option :tree,
+        :long => '--tree',
+        :boolean => true,
+        :description => "Show dependencies in a visual tree.  May show duplicates."
+      option :remote,
+        :long => '--remote',
+        :boolean => true,
+        :description => "List dependencies on the server instead of the local filesystem"
+
+      attr_accessor :exit_code
+
+      def run
+        if config[:recurse] == false && !config[:tree]
+          ui.error "--no-recurse requires --tree"
+          exit(1)
+        end
+        config[:recurse] = true if config[:recurse].nil?
+
+        @root = config[:remote] ? chef_fs : local_fs
+        dependencies = {}
+        pattern_args.each do |pattern|
+          Chef::ChefFS::FileSystem.list(@root, pattern).each do |entry|
+            if config[:tree]
+              print_dependencies_tree(entry, dependencies)
+            else
+              print_flattened_dependencies(entry, dependencies)
+            end
+          end
+        end
+        exit exit_code if exit_code
+      end
+
+      def print_flattened_dependencies(entry, dependencies)
+        if !dependencies[entry.path]
+          dependencies[entry.path] = get_dependencies(entry)
+          dependencies[entry.path].each do |child|
+            child_entry = Chef::ChefFS::FileSystem.resolve_path(@root, child)
+            print_flattened_dependencies(child_entry, dependencies)
+          end
+          output format_path(entry)
+        end
+      end
+
+      def print_dependencies_tree(entry, dependencies, printed = {}, depth = 0)
+        dependencies[entry.path] = get_dependencies(entry) if !dependencies[entry.path]
+        output "#{'  '*depth}#{format_path(entry)}"
+        if !printed[entry.path] && (config[:recurse] || depth == 0)
+          printed[entry.path] = true
+          dependencies[entry.path].each do |child|
+            child_entry = Chef::ChefFS::FileSystem.resolve_path(@root, child)
+            print_dependencies_tree(child_entry, dependencies, printed, depth+1)
+          end
+        end
+      end
+
+      def get_dependencies(entry)
+        begin
+          if entry.parent && entry.parent.path == '/cookbooks'
+            return entry.chef_object.metadata.dependencies.keys.map { |cookbook| "/cookbooks/#{cookbook}" }
+
+          elsif entry.parent && entry.parent.path == '/nodes'
+            node = JSON.parse(entry.read, :create_additions => false)
+            result = []
+            if node['chef_environment'] && node['chef_environment'] != '_default'
+              result << "/environments/#{node['chef_environment']}.json"
+            end
+            if node['run_list']
+              result += dependencies_from_runlist(node['run_list'])
+            end
+            result
+
+          elsif entry.parent && entry.parent.path == '/roles'
+            role = JSON.parse(entry.read, :create_additions => false)
+            result = []
+            if role['run_list']
+              dependencies_from_runlist(role['run_list']).each do |dependency|
+                result << dependency if !result.include?(dependency)
+              end
+            end
+            if role['env_run_lists']
+              role['env_run_lists'].each_pair do |env,run_list|
+                dependencies_from_runlist(run_list).each do |dependency|
+                  result << dependency if !result.include?(dependency)
+                end
+              end
+            end
+            result
+
+          elsif !entry.exists?
+            raise Chef::ChefFS::FileSystem::NotFoundError.new(entry)
+
+          else
+            []
+          end
+        rescue Chef::ChefFS::FileSystem::NotFoundError => e
+          ui.error "#{format_path(e.entry)}: No such file or directory"
+          self.exit_code = 2
+          []
+        end
+      end
+
+      def dependencies_from_runlist(run_list)
+        chef_run_list = Chef::RunList.new
+        chef_run_list.reset!(run_list)
+        chef_run_list.map do |run_list_item|
+          case run_list_item.type
+          when :role
+            "/roles/#{run_list_item.name}.json"
+          when :recipe
+            if run_list_item.name =~ /(.+)::[^:]*/
+              "/cookbooks/#{$1}"
+            else
+              "/cookbooks/#{run_list_item.name}"
+            end
+          else
+            raise "Unknown run list item type #{run_list_item.type}"
+          end
+        end
+      end
+    end
+  end
+end
+
diff --git a/lib/chef/knife/diff.rb b/lib/chef/knife/diff.rb
index 57d3bf3f0c..5a3a80544d 100644
--- a/lib/chef/knife/diff.rb
+++ b/lib/chef/knife/diff.rb
@@ -1,12 +1,13 @@
 require 'chef/chef_fs/knife'
-require 'chef/chef_fs/command_line'
 
 class Chef
   class Knife
     class Diff < Chef::ChefFS::Knife
       banner "knife diff PATTERNS"
 
-      common_options
+      deps do
+        require 'chef/chef_fs/command_line'
+      end
 
       option :recurse,
         :long => '--[no-]recurse',
@@ -24,6 +25,11 @@ class Chef
         :boolean => true,
         :description => "Only show names and statuses of modified files: Added, Deleted, Modified, and Type Changed."
 
+      option :diff_filter,
+        :long => '--diff-filter=[(A|D|M|T)...[*]]',
+        :description => "Select only files that are Added (A), Deleted (D), Modified (M), or have their type (i.e. regular file, directory) changed (T). Any combination of the filter characters (including none) can be used. When * (All-or-none) is added to the combination, all paths are selected if
+           there is any file that matches other criteria in the comparison; if there is no file that matches other criteria, nothing is selected."
+
       def run
         if config[:name_only]
           output_mode = :name_only
@@ -34,10 +40,21 @@ class Chef
         patterns = pattern_args_from(name_args.length > 0 ? name_args : [ "" ])
 
         # Get the matches (recursively)
-        patterns.each do |pattern|
-          Chef::ChefFS::CommandLine.diff(pattern, chef_fs, local_fs, config[:recurse] ? nil : 1, output_mode) do |diff|
-            puts diff
+        error = false
+        begin
+          patterns.each do |pattern|
+            found_error = Chef::ChefFS::CommandLine.diff_print(pattern, chef_fs, local_fs, config[:recurse] ? nil : 1, output_mode, proc { |entry| format_path(entry) }, config[:diff_filter], ui ) do |diff|
+              stdout.print diff
+            end
+            error = true if found_error
           end
+        rescue Chef::ChefFS::FileSystem::OperationFailedError => e
+          ui.error "Failed on #{format_path(e.entry)} in #{e.operation}: #{e.message}"
+          error = true
+        end
+
+        if error
+          exit 1
         end
       end
     end
diff --git a/lib/chef/knife/download.rb b/lib/chef/knife/download.rb
index dc2588b6b5..e8f26a74aa 100644
--- a/lib/chef/knife/download.rb
+++ b/lib/chef/knife/download.rb
@@ -1,12 +1,13 @@
 require 'chef/chef_fs/knife'
-require 'chef/chef_fs/command_line'
 
 class Chef
   class Knife
     class Download < Chef::ChefFS::Knife
       banner "knife download PATTERNS"
 
-      common_options
+      deps do
+        require 'chef/chef_fs/command_line'
+      end
 
       option :recurse,
         :long => '--[no-]recurse',
@@ -33,6 +34,12 @@ class Chef
         :default => false,
         :description => "Don't take action, only print what would happen"
 
+      option :diff,
+        :long => '--[no-]diff',
+        :boolean => true,
+        :default => true,
+        :description => 'Turn off to avoid uploading existing files; only new (and possibly deleted) files with --no-diff'
+
       def run
         if name_args.length == 0
           show_usage
@@ -40,8 +47,14 @@ class Chef
           exit 1
         end
 
+        error = false
         pattern_args.each do |pattern|
-          Chef::ChefFS::FileSystem.copy_to(pattern, chef_fs, local_fs, config[:recurse] ? nil : 1, config)
+          if Chef::ChefFS::FileSystem.copy_to(pattern, chef_fs, local_fs, config[:recurse] ? nil : 1, config, ui, proc { |entry| format_path(entry) })
+            error = true
+          end
+        end
+        if error
+          exit 1
         end
       end
     end
diff --git a/lib/chef/knife/edit.rb b/lib/chef/knife/edit.rb
new file mode 100644
index 0000000000..ea068cb250
--- /dev/null
+++ b/lib/chef/knife/edit.rb
@@ -0,0 +1,76 @@
+require 'chef/chef_fs/knife'
+
+class Chef
+  class Knife
+    class Edit < Chef::ChefFS::Knife
+      banner "knife edit [PATTERN1 ... PATTERNn]"
+
+      deps do
+        require 'chef/chef_fs/file_system'
+        require 'chef/chef_fs/file_system/not_found_error'
+      end
+
+      option :local,
+        :long => '--local',
+        :boolean => true,
+        :description => "Show local files instead of remote"
+
+      def run
+        # Get the matches (recursively)
+        error = false
+        pattern_args.each do |pattern|
+          Chef::ChefFS::FileSystem.list(config[:local] ? local_fs : chef_fs, pattern).each do |result|
+            if result.dir?
+              ui.error "#{format_path(result)}: is a directory" if pattern.exact_path
+              error = true
+            else
+              begin
+                new_value = edit_text(result.read, File.extname(result.name))
+                if new_value
+                  result.write(new_value)
+                  output "Updated #{format_path(result)}"
+                else
+                  output "#{format_path(result)} unchanged!"
+                end
+              rescue Chef::ChefFS::FileSystem::OperationNotAllowedError => e
+                ui.error "#{format_path(e.entry)}: #{e.reason}."
+                error = true
+              rescue Chef::ChefFS::FileSystem::NotFoundError => e
+                ui.error "#{format_path(e.entry)}: No such file or directory"
+                error = true
+              end
+            end
+          end
+        end
+        if error
+          exit 1
+        end
+      end
+
+      def edit_text(text, extension)
+        if (!config[:disable_editing])
+          file = Tempfile.new([ 'knife-edit-', extension ])
+          begin
+            # Write the text to a temporary file
+            file.open
+            file.write(text)
+            file.close
+
+            # Let the user edit the temporary file
+            if !system("#{config[:editor]} #{file.path}")
+              raise "Please set EDITOR environment variable"
+            end
+
+            file.open
+            result_text = file.read
+            return result_text if result_text != text
+
+          ensure
+            file.close!
+          end
+        end
+      end
+    end
+  end
+end
+
diff --git a/lib/chef/knife/list.rb b/lib/chef/knife/list.rb
index 30fcb5fa35..83d5c5a8c4 100644
--- a/lib/chef/knife/list.rb
+++ b/lib/chef/knife/list.rb
@@ -1,107 +1,151 @@
 require 'chef/chef_fs/knife'
-require 'chef/chef_fs/file_system'
 
 class Chef
   class Knife
     class List < Chef::ChefFS::Knife
-      banner "knife list [-dR] [PATTERN1 ... PATTERNn]"
+      banner "knife list [-dfR1p] [PATTERN1 ... PATTERNn]"
 
-      common_options
+      deps do
+        require 'chef/chef_fs/file_system'
+        require 'highline'
+      end
 
       option :recursive,
         :short => '-R',
         :boolean => true,
-        :description => "List directories recursively."
+        :description => "List directories recursively"
       option :bare_directories,
         :short => '-d',
         :boolean => true,
-        :description => "When directories match the pattern, do not show the directories' children."
+        :description => "When directories match the pattern, do not show the directories' children"
+      option :local,
+        :long => '--local',
+        :boolean => true,
+        :description => "List local directory instead of remote"
+      option :flat,
+        :short => '-f',
+        :long => '--flat',
+        :boolean => true,
+        :description => "Show a list of filenames rather than the prettified ls-like output normally produced"
+      option :one_column,
+        :short => '-1',
+        :boolean => true,
+        :description => "Show only one column of results"
+      option :trailing_slashes,
+        :short => '-p',
+        :boolean => true,
+        :description => "Show trailing slashes after directories"
+
+      attr_accessor :exit_code
 
       def run
         patterns = name_args.length == 0 ? [""] : name_args
 
         # Get the matches (recursively)
-        results = []
-        dir_results = []
-        pattern_args_from(patterns).each do |pattern|
-          Chef::ChefFS::FileSystem.list(chef_fs, pattern) do |result|
-            if result.dir? && !config[:bare_directories]
-              dir_results += add_dir_result(result)
-            elsif result.exists?
-              results << result
-            elsif pattern.exact_path
-              STDERR.puts "#{format_path(result.path)}: No such file or directory"
-            end
+        all_results = parallelize(pattern_args_from(patterns), :flatten => true) do |pattern|
+          pattern_results = Chef::ChefFS::FileSystem.list(config[:local] ? local_fs : chef_fs, pattern)
+          if pattern_results.first && !pattern_results.first.exists? && pattern.exact_path
+            ui.error "#{format_path(pattern_results.first)}: No such file or directory"
+            self.exit_code = 1
+          end
+          pattern_results
+        end
+
+        # Process directories
+        if !config[:bare_directories]
+          dir_results = parallelize(all_results.select { |result| result.dir? }, :flatten => true) do |result|
+            add_dir_result(result)
+          end.to_a
+        else
+          dir_results = []
+        end
+
+        # Process all other results
+        results = all_results.select { |result| result.exists? && (!result.dir? || config[:bare_directories]) }.to_a
+
+        # Flatten out directory results if necessary
+        if config[:flat]
+          dir_results.each do |result, children|
+            results += children
           end
+          dir_results = []
         end
 
+        # Sort by path for happy output
         results = results.sort_by { |result| result.path }
         dir_results = dir_results.sort_by { |result| result[0].path }
 
+        # Print!
         if results.length == 0 && dir_results.length == 1
           results = dir_results[0][1]
           dir_results = []
         end
 
         print_result_paths results
+        printed_something = results.length > 0
         dir_results.each do |result, children|
-          puts ""
-          puts "#{format_path(result.path)}:"
-          print_results(children.map { |result| result.name }.sort, "")
+          if printed_something
+            output ""
+          else
+            printed_something = true
+          end
+          output "#{format_path(result)}:"
+          print_results(children.map { |result| maybe_add_slash(result.name, result.dir?) }.sort, "")
         end
+
+        exit self.exit_code if self.exit_code
       end
 
       def add_dir_result(result)
         begin
           children = result.children.sort_by { |child| child.name }
-        rescue Chef::ChefFS::FileSystem::NotFoundError
-          STDERR.puts "#{format_path(result.path)}: No such file or directory"
+        rescue Chef::ChefFS::FileSystem::NotFoundError => e
+          ui.error "#{format_path(e.entry)}: No such file or directory"
           return []
         end
 
         result = [ [ result, children ] ]
         if config[:recursive]
-          children.each do |child|
-            if child.dir?
-              result += add_dir_result(child)
-            end
-          end
+          child_dirs = children.select { |child| child.dir? }
+          result += parallelize(child_dirs, :flatten => true) { |child| add_dir_result(child) }.to_a
         end
         result
       end
 
-      def list_dirs_recursive(children)
-        results = children.select { |child| child.dir? }.to_a
-        results.each do |child|
-          results += list_dirs_recursive(child.children)
-        end
-        results
-      end
-
       def print_result_paths(results, indent = "")
-        print_results(results.map { |result| format_path(result.path) }, indent)
+        print_results(results.map { |result| maybe_add_slash(format_path(result), result.dir?) }, indent)
       end
 
       def print_results(results, indent)
         return if results.length == 0
 
         print_space = results.map { |result| result.length }.max + 2
-        # TODO: tput cols is not cross platform
-        columns = $stdout.isatty ? Integer(`tput cols`) : 0
-        current_column = 0
+        if config[:one_column] || !stdout.isatty
+          columns = 0
+        else
+          columns = HighLine::SystemExtensions.terminal_size[0]
+        end
+        current_line = ''
         results.each do |result|
-          if current_column != 0 && current_column + print_space > columns
-            puts ""
-            current_column = 0
+          if current_line.length > 0 && current_line.length + print_space > columns
+            output current_line.rstrip
+            current_line = ''
           end
-          if current_column == 0
-            print indent
-            current_column += indent.length
+          if current_line.length == 0
+            current_line << indent
           end
-          print result + (' ' * (print_space - result.length))
-          current_column += print_space
+          current_line << result
+          current_line << (' ' * (print_space - result.length))
+        end
+        output current_line.rstrip if current_line.length > 0
+      end
+
+      def maybe_add_slash(path, is_dir)
+        if config[:trailing_slashes] && is_dir
+          "#{path}/"
+        else
+          path
         end
-        puts ""
       end
     end
   end
diff --git a/lib/chef/knife/raw.rb b/lib/chef/knife/raw.rb
index ad5d5f33ef..ee22d1ade5 100644
--- a/lib/chef/knife/raw.rb
+++ b/lib/chef/knife/raw.rb
@@ -1,21 +1,28 @@
-require 'json'
+require 'chef/knife'
 
 class Chef
   class Knife
     class Raw < Chef::Knife
       banner "knife raw REQUEST_PATH"
 
+      deps do
+        require 'json'
+        require 'chef/rest'
+        require 'chef/config'
+        require 'chef/chef_fs/raw_request'
+      end
+
       option :method,
         :long => '--method METHOD',
         :short => '-m METHOD',
         :default => "GET",
-        :description => "Request method (GET, POST, PUT or DELETE)"
+        :description => "Request method (GET, POST, PUT or DELETE).  Default: GET"
 
       option :pretty,
         :long => '--[no-]pretty',
         :boolean => true,
         :default => true,
-        :description => "Pretty-print JSON output"
+        :description => "Pretty-print JSON output.  Default: true"
 
       option :input,
         :long => '--input FILE',
@@ -39,70 +46,19 @@ class Chef
           data = IO.read(config[:input])
         end
         chef_rest = Chef::REST.new(Chef::Config[:chef_server_url])
-        puts api_request(chef_rest, config[:method].to_sym, chef_rest.create_url(name_args[0]), {}, data)
-      end
-
-      ACCEPT_ENCODING = "Accept-Encoding".freeze
-      ENCODING_GZIP_DEFLATE = "gzip;q=1.0,deflate;q=0.6,identity;q=0.3".freeze
-
-      def redirected_to(response)
-        return nil  unless response.kind_of?(Net::HTTPRedirection)
-        # Net::HTTPNotModified is undesired subclass of Net::HTTPRedirection so test for this
-        return nil  if response.kind_of?(Net::HTTPNotModified)
-        response['location']
-      end
-
-      def api_request(chef_rest, method, url, headers={}, data=false)
-        json_body = data
-#        json_body = data ? Chef::JSONCompat.to_json(data) : nil
-        # Force encoding to binary to fix SSL related EOFErrors
-        # cf. http://tickets.opscode.com/browse/CHEF-2363
-        # http://redmine.ruby-lang.org/issues/5233
-#        json_body.force_encoding(Encoding::BINARY) if json_body.respond_to?(:force_encoding)
-        headers = build_headers(chef_rest, method, url, headers, json_body)
-
-        chef_rest.retriable_rest_request(method, url, json_body, headers) do |rest_request|
-          response = rest_request.call {|r| r.read_body}
-
-          response_body = chef_rest.decompress_body(response)
-
-          if response.kind_of?(Net::HTTPSuccess)
-            if config[:pretty] && response['content-type'] =~ /json/
-              JSON.pretty_generate(JSON.parse(response_body, :create_additions => false))
-            else
-              response_body
-            end
-          elsif redirect_location = redirected_to(response)
-            raise "Redirected to #{create_url(redirect_location)}"
-            follow_redirect {api_request(:GET, create_url(redirect_location))}
-          else
-            # have to decompress the body before making an exception for it. But the body could be nil.
-            response.body.replace(chef_rest.decompress_body(response)) if response.body.respond_to?(:replace)
-
-            if response['content-type'] =~ /json/
-              exception = response_body
-              msg = "HTTP Request Returned #{response.code} #{response.message}: "
-              msg << (exception["error"].respond_to?(:join) ? exception["error"].join(", ") : exception["error"].to_s)
-              Chef::Log.info(msg)
-            end
-            puts response.body
-            response.error!
-          end
+        begin
+          output Chef::ChefFS::RawRequest.api_request(chef_rest, config[:method].to_sym, chef_rest.create_url(name_args[0]), {}, data)
+        rescue Timeout::Error => e
+          ui.error "Server timeout"
+          exit 1
+        rescue Net::HTTPServerException => e
+          ui.error "Server responded with error #{e.response.code} \"#{e.response.message}\""
+          ui.error "Error Body: #{e.response.body}" if e.response.body && e.response.body != ''
+          exit 1
         end
       end
 
-      def build_headers(chef_rest, method, url, headers={}, json_body=false, raw=false)
-#        headers                 = @default_headers.merge(headers)
-        #headers['Accept']       = "application/json" unless raw
-        headers['Accept']       = "application/json" unless raw
-        headers["Content-Type"] = 'application/json' if json_body
-        headers['Content-Length'] = json_body.bytesize.to_s if json_body
-        headers[Chef::REST::RESTRequest::ACCEPT_ENCODING] = Chef::REST::RESTRequest::ENCODING_GZIP_DEFLATE
-        headers.merge!(chef_rest.authentication_headers(method, url, json_body)) if chef_rest.sign_requests?
-        headers.merge!(Chef::Config[:custom_http_headers]) if Chef::Config[:custom_http_headers]
-        headers
-      end
-    end
+    end # class Raw
   end
 end
 
diff --git a/lib/chef/knife/show.rb b/lib/chef/knife/show.rb
index 7075315b08..b5e8aa9882 100644
--- a/lib/chef/knife/show.rb
+++ b/lib/chef/knife/show.rb
@@ -1,30 +1,53 @@
 require 'chef/chef_fs/knife'
-require 'chef/chef_fs/file_system'
 
 class Chef
   class Knife
     class Show < Chef::ChefFS::Knife
       banner "knife show [PATTERN1 ... PATTERNn]"
 
-      common_options
+      deps do
+        require 'chef/chef_fs/file_system'
+        require 'chef/chef_fs/file_system/not_found_error'
+      end
+
+      option :local,
+        :long => '--local',
+        :boolean => true,
+        :description => "Show local files instead of remote"
 
       def run
         # Get the matches (recursively)
-        pattern_args.each do |pattern|
-          Chef::ChefFS::FileSystem.list(chef_fs, pattern) do |result|
-            if result.dir?
-              STDERR.puts "#{result.path_for_printing}: is a directory" if pattern.exact_path
+        error = false
+        entry_values = parallelize(pattern_args, :flatten => true) do |pattern|
+          parallelize(Chef::ChefFS::FileSystem.list(config[:local] ? local_fs : chef_fs, pattern)) do |entry|
+            if entry.dir?
+              ui.error "#{format_path(entry)}: is a directory" if pattern.exact_path
+              error = true
+              nil
             else
               begin
-                value = result.read
-                puts "#{result.path_for_printing}:"
-                output(format_for_display(value))
-              rescue Chef::ChefFS::FileSystem::NotFoundError
-                STDERR.puts "#{result.path_for_printing}: No such file or directory"
+                [entry, entry.read]
+              rescue Chef::ChefFS::FileSystem::OperationNotAllowedError => e
+                ui.error "#{format_path(e.entry)}: #{e.reason}."
+                error = true
+                nil
+              rescue Chef::ChefFS::FileSystem::NotFoundError => e
+                ui.error "#{format_path(e.entry)}: No such file or directory"
+                error = true
+                nil
               end
             end
           end
         end
+        entry_values.each do |entry, value|
+          if entry
+            output "#{format_path(entry)}:"
+            output(format_for_display(value))
+          end
+        end
+        if error
+          exit 1
+        end
       end
     end
   end
diff --git a/lib/chef/knife/upload.rb b/lib/chef/knife/upload.rb
index bb1e365798..f72b6ea616 100644
--- a/lib/chef/knife/upload.rb
+++ b/lib/chef/knife/upload.rb
@@ -1,12 +1,13 @@
 require 'chef/chef_fs/knife'
-require 'chef/chef_fs/command_line'
 
 class Chef
   class Knife
     class Upload < Chef::ChefFS::Knife
       banner "knife upload PATTERNS"
 
-      common_options
+      deps do
+        require 'chef/chef_fs/command_line'
+      end
 
       option :recurse,
         :long => '--[no-]recurse',
@@ -24,7 +25,13 @@ class Chef
         :long => '--[no-]force',
         :boolean => true,
         :default => false,
-        :description => "Force upload of files even if they match (quicker and harmless, but doesn't print out what it changed)"
+        :description => "Force upload of files even if they match (quicker for many files).  Will overwrite frozen cookbooks."
+
+      option :freeze,
+        :long => '--[no-]freeze',
+        :boolean => true,
+        :default => false,
+        :description => "Freeze cookbooks that get uploaded."
 
       option :dry_run,
         :long => '--dry-run',
@@ -33,6 +40,12 @@ class Chef
         :default => false,
         :description => "Don't take action, only print what would happen"
 
+      option :diff,
+        :long => '--[no-]diff',
+        :boolean => true,
+        :default => true,
+        :description => 'Turn off to avoid uploading existing files; only new (and possibly deleted) files with --no-diff'
+
       def run
         if name_args.length == 0
           show_usage
@@ -40,8 +53,14 @@ class Chef
           exit 1
         end
 
+        error = false
         pattern_args.each do |pattern|
-          Chef::ChefFS::FileSystem.copy_to(pattern, local_fs, chef_fs, config[:recurse] ? nil : 1, config)
+          if Chef::ChefFS::FileSystem.copy_to(pattern, local_fs, chef_fs, config[:recurse] ? nil : 1, config, ui, proc { |entry| format_path(entry) })
+            error = true
+          end
+        end
+        if error
+          exit 1
         end
       end
     end
diff --git a/lib/chef/knife/xargs.rb b/lib/chef/knife/xargs.rb
new file mode 100644
index 0000000000..be6db9d64f
--- /dev/null
+++ b/lib/chef/knife/xargs.rb
@@ -0,0 +1,265 @@
+require 'chef/chef_fs/knife'
+
+class Chef
+  class Knife
+    class Xargs < Chef::ChefFS::Knife
+      banner "knife xargs [COMMAND]"
+
+      deps do
+        require 'chef/chef_fs/file_system'
+        require 'chef/chef_fs/file_system/not_found_error'
+      end
+
+      # TODO modify to remote-only / local-only pattern (more like delete)
+      option :local,
+        :long => '--local',
+        :boolean => true,
+        :description => "Xargs local files instead of remote"
+
+      option :patterns,
+        :long => '--pattern [PATTERN]',
+        :short => '-p [PATTERN]',
+        :description => "Pattern on command line (if these are not specified, a list of patterns is expected on standard input).  Multiple patterns may be passed in this way.",
+        :arg_arity => [1,-1]
+
+      option :diff,
+        :long => '--[no-]diff',
+        :default => true,
+        :boolean => true,
+        :description => "Whether to show a diff when files change (default: true)"
+
+      option :dry_run,
+        :long => '--dry-run',
+        :boolean => true,
+        :description => "Prevents changes from actually being uploaded to the server."
+
+      option :force,
+        :long => '--[no-]force',
+        :boolean => true,
+        :default => false,
+        :description => "Force upload of files even if they are not changed (quicker and harmless, but doesn't print out what it changed)"
+
+      option :replace_first,
+        :long => '--replace-first REPLACESTR',
+        :short => '-J REPLACESTR',
+        :description => "String to replace with filenames.  -J will only replace the FIRST occurrence of the replacement string."
+
+      option :replace_all,
+        :long => '--replace REPLACESTR',
+        :short => '-I REPLACESTR',
+        :description => "String to replace with filenames.  -I will replace ALL occurrence of the replacement string."
+
+      option :max_arguments_per_command,
+        :long => '--max-args MAXARGS',
+        :short => '-n MAXARGS',
+        :description => "Maximum number of arguments per command line."
+
+      option :max_command_line,
+        :long => '--max-chars LENGTH',
+        :short => '-s LENGTH',
+        :description => "Maximum size of command line, in characters"
+
+      option :verbose_commands,
+        :short => '-t',
+        :description => "Print command to be run on the command line"
+
+      option :null_separator,
+        :short => '-0',
+        :boolean => true,
+        :description => "Use the NULL character (\0) as a separator, instead of whitespace"
+
+      def run
+        error = false
+        # Get the matches (recursively)
+        files = []
+        pattern_args_from(get_patterns).each do |pattern|
+          Chef::ChefFS::FileSystem.list(config[:local] ? local_fs : chef_fs, pattern).each do |result|
+            if result.dir?
+              # TODO option to include directories
+              ui.warn "#{format_path(result)}: is a directory.  Will not run #{command} on it."
+            else
+              files << result
+              ran = false
+
+              # If the command would be bigger than max command line, back it off a bit
+              # and run a slightly smaller command (with one less arg)
+              if config[:max_command_line]
+                command, tempfiles = create_command(files)
+                begin
+                  if command.length > config[:max_command_line].to_i
+                    if files.length > 1
+                      command, tempfiles_minus_one = create_command(files[0..-2])
+                      begin
+                        error = true if xargs_files(command, tempfiles_minus_one)
+                        files = [ files[-1] ]
+                        ran = true
+                      ensure
+                        destroy_tempfiles(tempfiles)
+                      end
+                    else
+                      error = true if xargs_files(command, tempfiles)
+                      files = [ ]
+                      ran = true
+                    end
+                  end
+                ensure
+                  destroy_tempfiles(tempfiles)
+                end
+              end
+
+              # If the command has hit the limit for the # of arguments, run it
+              if !ran && config[:max_arguments_per_command] && files.size >= config[:max_arguments_per_command].to_i
+                command, tempfiles = create_command(files)
+                begin
+                  error = true if xargs_files(command, tempfiles)
+                  files = []
+                  ran = true
+                ensure
+                  destroy_tempfiles(tempfiles)
+                end
+              end
+            end
+          end
+        end
+
+        # Any leftovers commands shall be run
+        if files.size > 0
+          command, tempfiles = create_command(files)
+          begin
+            error = true if xargs_files(command, tempfiles)
+          ensure
+            destroy_tempfiles(tempfiles)
+          end
+        end
+
+        if error
+          exit 1
+        end
+      end
+
+      def get_patterns
+        if config[:patterns]
+          [ config[:patterns] ].flatten
+        elsif config[:null_separator]
+          stdin.binmode
+          stdin.read.split("\000")
+        else
+          stdin.read.split(/\s+/)
+        end
+      end
+
+      def create_command(files)
+        command = name_args.join(' ')
+
+        # Create the (empty) tempfiles
+        tempfiles = {}
+        begin
+          # Create the temporary files
+          files.each do |file|
+            tempfile = Tempfile.new(file.name)
+            tempfiles[tempfile] = { :file => file }
+          end
+        rescue
+          destroy_tempfiles(files)
+          raise
+        end
+
+        # Create the command
+        paths = tempfiles.keys.map { |tempfile| tempfile.path }.join(' ')
+        if config[:replace_all]
+          final_command = command.gsub(config[:replace_all], paths)
+        elsif config[:replace_first]
+          final_command = command.sub(config[:replace_first], paths)
+        else
+          final_command = "#{command} #{paths}"
+        end
+
+        [final_command, tempfiles]
+      end
+
+      def destroy_tempfiles(tempfiles)
+        # Unlink the files now that we're done with them
+        tempfiles.keys.each { |tempfile| tempfile.close! }
+      end
+
+      def xargs_files(command, tempfiles)
+        error = false
+        # Create the temporary files
+        tempfiles.each_pair do |tempfile, file|
+          begin
+            value = file[:file].read
+            file[:value] = value
+            tempfile.open
+            tempfile.write(value)
+            tempfile.close
+          rescue Chef::ChefFS::FileSystem::OperationNotAllowedError => e
+            ui.error "#{format_path(e.entry)}: #{e.reason}."
+            error = true
+            tempfile.close!
+            tempfiles.delete(tempfile)
+            next
+          rescue Chef::ChefFS::FileSystem::NotFoundError => e
+            ui.error "#{format_path(e.entry)}: No such file or directory"
+            error = true
+            tempfile.close!
+            tempfiles.delete(tempfile)
+            next
+          end
+        end
+
+        return error if error && tempfiles.size == 0
+
+        # Run the command
+        if config[:verbose_commands] || Chef::Config[:verbosity] && Chef::Config[:verbosity] >= 1
+          output sub_filenames(command, tempfiles)
+        end
+        command_output = `#{command}`
+        command_output = sub_filenames(command_output, tempfiles)
+        stdout.write command_output
+
+        # Check if the output is different
+        tempfiles.each_pair do |tempfile, file|
+          # Read the new output
+          new_value = IO.binread(tempfile.path)
+
+          # Upload the output if different
+          if config[:force] || new_value != file[:value]
+            if config[:dry_run]
+              output "Would update #{format_path(file[:file])}"
+            else
+              file[:file].write(new_value)
+              output "Updated #{format_path(file[:file])}"
+            end
+          end
+
+          # Print a diff of what was uploaded
+          if config[:diff] && new_value != file[:value]
+            old_file = Tempfile.open(file[:file].name)
+            begin
+              old_file.write(file[:value])
+              old_file.close
+
+              diff = `diff -u #{old_file.path} #{tempfile.path}`
+              diff.gsub!(old_file.path, "#{format_path(file[:file])} (old)")
+              diff.gsub!(tempfile.path, "#{format_path(file[:file])} (new)")
+              stdout.write diff
+            ensure
+              old_file.close!
+            end
+          end
+        end
+
+        error
+      end
+
+      def sub_filenames(str, tempfiles)
+        tempfiles.each_pair do |tempfile, file|
+          str = str.gsub(tempfile.path, format_path(file[:file]))
+        end
+        str
+      end
+
+    end
+  end
+end
+