[CORE] Bypass chef_object inflation and normalize raw requests instead

  - Factored out api_request() from knife_essentials to a class method
    available everywhere
  - Added #raw_request to handle GET requests without chef object inflation
  - Added RestListEntry#chef_hash to pull a raw_request()
  - RestListEntry#read will now normalize from #chef_hash
  - Added RestListDir#chef_collection to make it easier to customize and test
  - Updated unit tests to mock #chef_hash and #chef_collection instead of using a
    @rest mock, where appropriate

7 files changed, 109 insertions, 80 deletions

diff --git a/lib/chef/chef_fs/data_handler/data_handler_base.rb b/lib/chef/chef_fs/data_handler/data_handler_base.rb
index 0d007cf24c..3fe539b29f 100644
--- a/lib/chef/chef_fs/data_handler/data_handler_base.rb
+++ b/lib/chef/chef_fs/data_handler/data_handler_base.rb
@@ -106,7 +106,8 @@ class Chef
           end
           result
         end
-      end
+
+      end # class DataHandlerBase
     end
   end
 end
diff --git a/lib/chef/chef_fs/file_system/base_fs_object.rb b/lib/chef/chef_fs/file_system/base_fs_object.rb
index e73ebae75a..e86d2f029a 100644
--- a/lib/chef/chef_fs/file_system/base_fs_object.rb
+++ b/lib/chef/chef_fs/file_system/base_fs_object.rb
@@ -104,6 +104,11 @@ class Chef
           []
         end
 
+        def chef_hash
+          raise NotFoundError.new(self) if !exists?
+          nil
+        end
+
         # Expand this entry into a chef object (Chef::Role, ::Node, etc.)
         def chef_object
           raise NotFoundError.new(self) if !exists?
@@ -172,7 +177,76 @@ class Chef
         # Important directory attributes: name, parent, path, root
         # Overridable attributes: dir?, child(name), path_for_printing
         # Abstract: read, write, delete, children, can_have_child?, create_child, compare_to
-      end
+
+        # Consider putting this into a concern module and including it instead
+        def raw_request(_api_path)
+          self.class.api_request(rest, :GET, rest.create_url(_api_path), {}, false)
+        end
+
+
+        class << self
+          # Copied so that it does not automatically inflate an object
+          # This is also used by knife raw_essentials
+
+          ACCEPT_ENCODING = "Accept-Encoding".freeze
+          ENCODING_GZIP_DEFLATE = "gzip;q=1.0,deflate;q=0.6,identity;q=0.3".freeze
+
+          def redirected_to(response)
+            return nil  unless response.kind_of?(Net::HTTPRedirection)
+            # Net::HTTPNotModified is undesired subclass of Net::HTTPRedirection so test for this
+            return nil  if response.kind_of?(Net::HTTPNotModified)
+            response['location']
+          end
+
+
+          def build_headers(chef_rest, method, url, headers={}, json_body=false, raw=false)
+            #        headers                 = @default_headers.merge(headers)
+            #headers['Accept']       = "application/json" unless raw
+            headers['Accept']       = "application/json" unless raw
+            headers["Content-Type"] = 'application/json' if json_body
+            headers['Content-Length'] = json_body.bytesize.to_s if json_body
+            headers[Chef::REST::RESTRequest::ACCEPT_ENCODING] = Chef::REST::RESTRequest::ENCODING_GZIP_DEFLATE
+            headers.merge!(chef_rest.authentication_headers(method, url, json_body)) if chef_rest.sign_requests?
+            headers.merge!(Chef::Config[:custom_http_headers]) if Chef::Config[:custom_http_headers]
+            headers
+          end
+
+          def api_request(chef_rest, method, url, headers={}, data=false)
+            json_body = data
+            #        json_body = data ? Chef::JSONCompat.to_json(data) : nil
+            # Force encoding to binary to fix SSL related EOFErrors
+            # cf. http://tickets.opscode.com/browse/CHEF-2363
+            # http://redmine.ruby-lang.org/issues/5233
+            #        json_body.force_encoding(Encoding::BINARY) if json_body.respond_to?(:force_encoding)
+            headers = build_headers(chef_rest, method, url, headers, json_body)
+
+            chef_rest.retriable_rest_request(method, url, json_body, headers) do |rest_request|
+              response = rest_request.call {|r| r.read_body}
+
+              response_body = chef_rest.decompress_body(response)
+
+              if response.kind_of?(Net::HTTPSuccess)
+                  response_body
+              elsif redirect_location = redirected_to(response)
+                raise "Redirected to #{create_url(redirect_location)}"
+                follow_redirect {api_request(:GET, create_url(redirect_location))}
+              else
+                # have to decompress the body before making an exception for it. But the body could be nil.
+                response.body.replace(chef_rest.decompress_body(response)) if response.body.respond_to?(:replace)
+
+                if response['content-type'] =~ /json/
+                  exception = response_body
+                  msg = "HTTP Request Returned #{response.code} #{response.message}: "
+                  msg << (exception["error"].respond_to?(:join) ? exception["error"].join(", ") : exception["error"].to_s)
+                  Chef::Log.info(msg)
+                end
+                response.error!
+              end
+            end
+          end
+        end
+
+      end # class BaseFsObject
     end
   end
 end
diff --git a/lib/chef/chef_fs/file_system/data_bags_dir.rb b/lib/chef/chef_fs/file_system/data_bags_dir.rb
index 5fea67f552..3eabb0f822 100644
--- a/lib/chef/chef_fs/file_system/data_bags_dir.rb
+++ b/lib/chef/chef_fs/file_system/data_bags_dir.rb
@@ -34,7 +34,7 @@ class Chef
 
         def children
           begin
-            @children ||= rest.get_rest(api_path).keys.sort.map do |entry|
+            @children ||= chef_collection.keys.sort.map do |entry|
               DataBagDir.new(entry, self, true)
             end
           rescue Net::HTTPServerException
@@ -46,6 +46,10 @@ class Chef
           end
         end
 
+        def chef_collection
+          rest.get_rest(api_path)
+        end
+
         def can_have_child?(name, is_dir)
           is_dir
         end
diff --git a/lib/chef/chef_fs/file_system/nodes_dir.rb b/lib/chef/chef_fs/file_system/nodes_dir.rb
index 311ebe8f61..481cd16261 100644
--- a/lib/chef/chef_fs/file_system/nodes_dir.rb
+++ b/lib/chef/chef_fs/file_system/nodes_dir.rb
@@ -29,20 +29,14 @@ class Chef
           super("nodes", parent, nil, Chef::ChefFS::DataHandler::NodeDataHandler.new)
         end
 
-        # Override children to respond to environment
-        # TODO let's not do this mmkay
-        def children
-          @children ||= begin
-            env_api_path = environment ? "environments/#{environment}/#{api_path}" : api_path
-            rest.get_rest(env_api_path).keys.sort.map { |key| RestListEntry.new("#{key}.json", self, true) }
-          rescue Net::HTTPServerException
-            if $!.response.code == "404"
-              raise Chef::ChefFS::FileSystem::NotFoundError.new(self, $!)
-            else
-              raise
-            end
-          end
-      end
+        # Override to respond to environment
+        def chef_collection
+          rest.get_rest(env_api_path)
+        end
+
+        def env_api_path
+          environment ? "environments/#{environment}/#{api_path}" : api_path
+        end
       end
     end
   end
diff --git a/lib/chef/chef_fs/file_system/rest_list_dir.rb b/lib/chef/chef_fs/file_system/rest_list_dir.rb
index a44cbbdbb5..f3e11564e4 100644
--- a/lib/chef/chef_fs/file_system/rest_list_dir.rb
+++ b/lib/chef/chef_fs/file_system/rest_list_dir.rb
@@ -45,7 +45,7 @@ class Chef
 
         def children
           begin
-            @children ||= rest.get_rest(api_path).keys.sort.map do |key|
+            @children ||= chef_collection.keys.sort.map do |key|
               _make_child_entry("#{key}.json", true)
             end
           rescue Net::HTTPServerException => e
@@ -57,6 +57,10 @@ class Chef
           end
         end
 
+        def chef_collection
+          rest.get_rest(api_path)
+        end
+
         def identity_key
           'name'
         end
diff --git a/lib/chef/chef_fs/file_system/rest_list_entry.rb b/lib/chef/chef_fs/file_system/rest_list_entry.rb
index 1ef5162b57..459e61ec0d 100644
--- a/lib/chef/chef_fs/file_system/rest_list_entry.rb
+++ b/lib/chef/chef_fs/file_system/rest_list_entry.rb
@@ -75,7 +75,17 @@ class Chef
 
         def read
           # Minimize the value so the results don't look terrible
-          Chef::JSONCompat.to_json_pretty(minimize_value(chef_object.to_hash))
+          Chef::JSONCompat.to_json_pretty(minimize_value(chef_hash))
+        end
+
+        def chef_hash
+          JSON.parse(raw_request(api_path), :create_additions => false)
+        rescue Net::HTTPServerException => e
+          if $!.response.code == "404"
+            raise Chef::ChefFS::FileSystem::NotFoundError.new(self, $!)
+          else
+            raise Chef::ChefFS::FileSystem::OperationFailedError.new(:read, self, e)
+          end
         end
 
         def chef_object
diff --git a/lib/chef/knife/raw.rb b/lib/chef/knife/raw.rb
index 76ffd73250..4abc7e6756 100644
--- a/lib/chef/knife/raw.rb
+++ b/lib/chef/knife/raw.rb
@@ -1,4 +1,5 @@
 require 'json'
+require 'chef/chef_fs/data_handler/data_handler_base'
 
 class Chef
   class Knife
@@ -40,7 +41,7 @@ class Chef
         end
         chef_rest = Chef::REST.new(Chef::Config[:chef_server_url])
         begin
-          output api_request(chef_rest, config[:method].to_sym, chef_rest.create_url(name_args[0]), {}, data)
+          output Chef::ChefFS::FileSystem::BaseFSObject.api_request(chef_rest, config[:method].to_sym, chef_rest.create_url(name_args[0]), {}, data)
         rescue Net::HTTPServerException => e
           ui.error "Server responded with error #{e.response.code} \"#{e.response.message}\""
           ui.error "Error Body: #{e.response.body}" if e.response.body && e.response.body != ''
@@ -48,66 +49,7 @@ class Chef
         end
       end
 
-      ACCEPT_ENCODING = "Accept-Encoding".freeze
-      ENCODING_GZIP_DEFLATE = "gzip;q=1.0,deflate;q=0.6,identity;q=0.3".freeze
-
-      def redirected_to(response)
-        return nil  unless response.kind_of?(Net::HTTPRedirection)
-        # Net::HTTPNotModified is undesired subclass of Net::HTTPRedirection so test for this
-        return nil  if response.kind_of?(Net::HTTPNotModified)
-        response['location']
-      end
-
-      def api_request(chef_rest, method, url, headers={}, data=false)
-        json_body = data
-#        json_body = data ? Chef::JSONCompat.to_json(data) : nil
-        # Force encoding to binary to fix SSL related EOFErrors
-        # cf. http://tickets.opscode.com/browse/CHEF-2363
-        # http://redmine.ruby-lang.org/issues/5233
-#        json_body.force_encoding(Encoding::BINARY) if json_body.respond_to?(:force_encoding)
-        headers = build_headers(chef_rest, method, url, headers, json_body)
-
-        chef_rest.retriable_rest_request(method, url, json_body, headers) do |rest_request|
-          response = rest_request.call {|r| r.read_body}
-
-          response_body = chef_rest.decompress_body(response)
-
-          if response.kind_of?(Net::HTTPSuccess)
-            if config[:pretty] && response['content-type'] =~ /json/
-              JSON.pretty_generate(JSON.parse(response_body, :create_additions => false))
-            else
-              response_body
-            end
-          elsif redirect_location = redirected_to(response)
-            raise "Redirected to #{create_url(redirect_location)}"
-            follow_redirect {api_request(:GET, create_url(redirect_location))}
-          else
-            # have to decompress the body before making an exception for it. But the body could be nil.
-            response.body.replace(chef_rest.decompress_body(response)) if response.body.respond_to?(:replace)
-
-            if response['content-type'] =~ /json/
-              exception = response_body
-              msg = "HTTP Request Returned #{response.code} #{response.message}: "
-              msg << (exception["error"].respond_to?(:join) ? exception["error"].join(", ") : exception["error"].to_s)
-              Chef::Log.info(msg)
-            end
-            response.error!
-          end
-        end
-      end
-
-      def build_headers(chef_rest, method, url, headers={}, json_body=false, raw=false)
-#        headers                 = @default_headers.merge(headers)
-        #headers['Accept']       = "application/json" unless raw
-        headers['Accept']       = "application/json" unless raw
-        headers["Content-Type"] = 'application/json' if json_body
-        headers['Content-Length'] = json_body.bytesize.to_s if json_body
-        headers[Chef::REST::RESTRequest::ACCEPT_ENCODING] = Chef::REST::RESTRequest::ENCODING_GZIP_DEFLATE
-        headers.merge!(chef_rest.authentication_headers(method, url, json_body)) if chef_rest.sign_requests?
-        headers.merge!(Chef::Config[:custom_http_headers]) if Chef::Config[:custom_http_headers]
-        headers
-      end
-    end
+    end # class Raw
   end
 end