Merge pull request #1912 from jessehu/CHEF-ISSUE-1904

retry on HTTP 50X Error when calling Chef REST API

2 files changed, 23 insertions, 13 deletions

diff --git a/lib/chef/http.rb b/lib/chef/http.rb
index abc47f636e..7f2d00157b 100644
--- a/lib/chef/http.rb
+++ b/lib/chef/http.rb
@@ -289,11 +289,26 @@ class Chef
     def retrying_http_errors(url)
       http_attempts = 0
       begin
-        http_attempts += 1
-
-        yield
-
+        loop do
+          http_attempts += 1
+          response, request, return_value = yield
+          # handle HTTP 50X Error
+          if response.kind_of?(Net::HTTPServerError)
+            if http_retry_count - http_attempts + 1 > 0
+              sleep_time = 1 + (2 ** http_attempts) + rand(2 ** http_attempts)
+              Chef::Log.error("Server returned error #{response.code} for #{url}, retrying #{http_attempts}/#{http_retry_count} in #{sleep_time}s")
+              sleep(sleep_time)
+              redo
+            end
+          end
+          return [response, request, return_value]
+        end
       rescue SocketError, Errno::ETIMEDOUT => e
+        if http_retry_count - http_attempts + 1 > 0
+          Chef::Log.error("Error connecting to #{url}, retry #{http_attempts}/#{http_retry_count}")
+          sleep(http_retry_delay)
+          retry
+        end
         e.message.replace "Error connecting to #{url} - #{e.message}"
         raise e
       rescue Errno::ECONNREFUSED
@@ -310,14 +325,6 @@ class Chef
           retry
         end
         raise Timeout::Error, "Timeout connecting to #{url}, giving up"
-      rescue Net::HTTPFatalError => e
-        if http_retry_count - http_attempts + 1 > 0
-          sleep_time = 1 + (2 ** http_attempts) + rand(2 ** http_attempts)
-          Chef::Log.error("Server returned error for #{url}, retrying #{http_attempts}/#{http_retry_count} in #{sleep_time}s")
-          sleep(sleep_time)
-          retry
-        end
-        raise
       end
     end
 
diff --git a/spec/unit/rest_spec.rb b/spec/unit/rest_spec.rb
index 3ad822193b..424fd12ee9 100644
--- a/spec/unit/rest_spec.rb
+++ b/spec/unit/rest_spec.rb
@@ -526,9 +526,12 @@ describe Chef::REST do
             http_response.stub(:read_body)
             http_response
           end
-          it "throws an exception" do
+
+          it "retries then throws an exception" do
             rest.stub(:sleep)
             expect {rest.request(:GET, url)}.to raise_error(Net::HTTPFatalError)
+            count = Chef::Config[:http_retry_count]
+            expect(log_stringio.string).to match(Regexp.escape("ERROR: Server returned error 500 for #{url}, retrying #{count}/#{count}"))
           end
         end
       end