CHEF-5041: check for transfer-encoding header

- skip content-length check in the presence of a transfer-encoding
  header
- also adds debugging and some refactoring

3 files changed, 35 insertions, 13 deletions

diff --git a/lib/chef/http.rb b/lib/chef/http.rb
index 78c47735d2..06de37fc69 100644
--- a/lib/chef/http.rb
+++ b/lib/chef/http.rb
@@ -210,18 +210,21 @@ class Chef
 
     def apply_request_middleware(method, url, headers, data)
       middlewares.inject([method, url, headers, data]) do |req_data, middleware|
+        Chef::Log.debug "calling handle_request for the #{middleware.class} middleware"
         middleware.handle_request(*req_data)
       end
     end
 
     def apply_response_middleware(response, rest_request, return_value)
       middlewares.reverse.inject([response, rest_request, return_value]) do |res_data, middleware|
+        Chef::Log.debug "calling handle_response for the #{middleware.class} middleware"
         middleware.handle_response(*res_data)
       end
     end
 
     def apply_stream_complete_middleware(response, rest_request, return_value)
       middlewares.reverse.inject([response, rest_request, return_value]) do |res_data, middleware|
+        Chef::Log.debug "calling handle_stream_complete for the #{middleware.class} middleware"
         middleware.handle_stream_complete(*res_data)
       end
     end
diff --git a/lib/chef/http/decompressor.rb b/lib/chef/http/decompressor.rb
index 78af47798c..c760dad11c 100644
--- a/lib/chef/http/decompressor.rb
+++ b/lib/chef/http/decompressor.rb
@@ -26,6 +26,7 @@ class Chef
     class Decompressor
       class NoopInflater
         def inflate(chunk)
+          Chef::Log.debug "decompressing compressed chunk"
           chunk
         end
         alias :handle_chunk :inflate
@@ -98,10 +99,10 @@ class Chef
         else
           case response[CONTENT_ENCODING]
           when GZIP
-            Chef::Log.debug "decompressing gzip stream"
+            Chef::Log.debug "initializing gzip stream deflator"
             GzipInflater.new
           when DEFLATE
-            Chef::Log.debug "decompressing inflate stream"
+            Chef::Log.debug "initializing deflate stream deflator"
             DeflateInflater.new
           else
             NoopInflater.new
diff --git a/lib/chef/http/validate_content_length.rb b/lib/chef/http/validate_content_length.rb
index 49f1738d42..834bb023e8 100644
--- a/lib/chef/http/validate_content_length.rb
+++ b/lib/chef/http/validate_content_length.rb
@@ -32,10 +32,12 @@ class Chef
         attr_accessor :content_length
 
         def initialize
+          Chef::Log.debug "initializing stream counter for response body size"
           @content_length = 0
         end
 
         def handle_chunk(chunk)
+          Chef::Log.debug "adding chunk to response body size: #{@content_length}"
           @content_length += chunk.bytesize
           chunk
         end
@@ -49,21 +51,15 @@ class Chef
       end
 
       def handle_response(http_response, rest_request, return_value)
-        unless http_response['content-length']
-          Chef::Log.debug("HTTP server did not include a Content-Length header in response, cannot identify truncated downloads.")
-          return [http_response, rest_request, return_value]
-        end
-        validate(response_content_length(http_response), http_response.body.bytesize)
+        validate(http_response, http_response.body.bytesize)
         return [http_response, rest_request, return_value]
       end
 
       def handle_stream_complete(http_response, rest_request, return_value)
-        if http_response['content-length'].nil?
-          Chef::Log.debug("HTTP server did not include a Content-Length header in response, cannot idenfity streamed download.")
-        elsif @content_length_counter.nil?
+        if @content_length_counter.nil?
           Chef::Log.debug("No content-length information collected for the streamed download, cannot identify streamed download.")
         else
-          validate(response_content_length(http_response), @content_length_counter.content_length)
+          validate(http_response, @content_length_counter.content_length)
         end
         return [http_response, rest_request, return_value]
       end
@@ -73,6 +69,7 @@ class Chef
       end
 
       private
+
       def response_content_length(response)
         if response['content-length'].is_a?(Array)
           response['content-length'].first.to_i
@@ -81,12 +78,33 @@ class Chef
         end
       end
 
-      def validate(content_length, response_length)
+      def validate(http_response, response_length)
+        content_length    = response_content_length(http_response)
+        transfer_encoding = http_response['transfer_encoding']
+        content_encoding  = http_response['content_encoding']
+
+        Chef::Log.debug "Attempting to validate the Content-Length header of the response"
         Chef::Log.debug "Content-Length header = #{content_length}"
-        Chef::Log.debug "Response body length = #{response_length}"
+        Chef::Log.debug "Transfer-Encoding header = #{transfer_encoding}"
+        Chef::Log.debug "Content-Encoding header = #{content_encoding}"
+        Chef::Log.debug "Actual response body length = #{response_length}"
+
+        if content_length.nil?
+          Chef::Log.debug "HTTP server did not include a Content-Length header in response, cannot identify truncated downloads."
+          return true
+        end
+
+        # if Transfer-Encoding is set the RFC states that we must ignore the Content-Length field
+        # CHEF-5041: some proxies uncompress gzip content, leave the incorrect content-length, but set the transfer-encoding field
+        unless transfer_encoding.nil?
+          Chef::Log.debug "Transfer-Encoding header is set, skipping Content-Length check."
+          return true
+        end
+
         if response_length != content_length
           raise Chef::Exceptions::ContentLengthMismatch.new(response_length, content_length)
         end
+
         true
       end
     end