SERVER-35800 retry getting build_id and test_id from logkeeper

3 files changed, 23 insertions, 6 deletions

diff --git a/buildscripts/resmokelib/logging/buildlogger.py b/buildscripts/resmokelib/logging/buildlogger.py
index 2e48101d517..c4697574d4f 100644
--- a/buildscripts/resmokelib/logging/buildlogger.py
+++ b/buildscripts/resmokelib/logging/buildlogger.py
@@ -282,7 +282,7 @@ class BuildloggerServer(object):
         build_num = int(self.config["build_num"])
 
         handler = handlers.HTTPHandler(url_root=_config.BUILDLOGGER_URL, username=username,
-                                       password=password)
+                                       password=password, should_retry=True)
 
         response = handler.post(CREATE_BUILD_ENDPOINT, data={
             "builder": builder,
@@ -297,7 +297,7 @@ class BuildloggerServer(object):
         """Return a new test id for sending test logs to."""
         handler = handlers.HTTPHandler(url_root=_config.BUILDLOGGER_URL,
                                        username=self.config["username"],
-                                       password=self.config["password"])
+                                       password=self.config["password"], should_retry=True)
 
         endpoint = CREATE_TEST_ENDPOINT % {"build_id": build_id}
         response = handler.post(
diff --git a/buildscripts/resmokelib/logging/handlers.py b/buildscripts/resmokelib/logging/handlers.py
index 982a2f38b6e..d67bf13f724 100644
--- a/buildscripts/resmokelib/logging/handlers.py
+++ b/buildscripts/resmokelib/logging/handlers.py
@@ -9,6 +9,7 @@ import threading
 import warnings
 
 import requests
+import requests.adapters
 import requests.auth
 
 try:
@@ -17,6 +18,8 @@ except ImportError:
     # Versions of the requests package prior to 1.2.0 did not vendor the urllib3 package.
     urllib3_exceptions = None
 
+import urllib3.util.retry as urllib3_retry
+
 from . import flush
 from .. import utils
 
@@ -159,11 +162,24 @@ class BufferedHandler(logging.Handler):
 class HTTPHandler(object):
     """A class which sends data to a web server using POST requests."""
 
-    def __init__(self, url_root, username, password):
+    def __init__(self, url_root, username, password, should_retry=False):
         """Initialize the handler with the necessary authentication credentials."""
 
         self.auth_handler = requests.auth.HTTPBasicAuth(username, password)
 
+        self.session = requests.Session()
+
+        if should_retry:
+            retry_status = [500, 502, 503, 504]  # Retry for these statuses.
+            retry = urllib3_retry.Retry(
+                backoff_factor=0.1,  # Enable backoff starting at 0.1s.
+                method_whitelist=False,  # Support all HTTP verbs.
+                status_forcelist=retry_status)
+
+            adapter = requests.adapters.HTTPAdapter(max_retries=retry)
+            self.session.mount('http://', adapter)
+            self.session.mount('https://', adapter)
+
         self.url_root = url_root
 
     def _make_url(self, endpoint):
@@ -205,8 +221,9 @@ class HTTPHandler(object):
                     # that defined InsecureRequestWarning.
                     pass
 
-            response = requests.post(url, data=data, headers=headers, timeout=timeout_secs,
-                                     auth=self.auth_handler, verify=should_validate_certificates)
+            response = self.session.post(url, data=data, headers=headers, timeout=timeout_secs,
+                                         auth=self.auth_handler,
+                                         verify=should_validate_certificates)
 
         response.raise_for_status()
 
diff --git a/buildscripts/resmokelib/requirements.txt b/buildscripts/resmokelib/requirements.txt
index 0f32c9d9b64..9d7bd449144 100644
--- a/buildscripts/resmokelib/requirements.txt
+++ b/buildscripts/resmokelib/requirements.txt
@@ -3,5 +3,5 @@ pymongo >= 3.0
 pypiwin32 == 219 ; sys_platform == "win32" and python_version < "3"
 pypiwin32 == 223 ; sys_platform == "win32" and python_version > "3"
 PyYAML == 3.11
-requests >= 2.0.0
+requests >= 2.16.1
 subprocess32 >= 3.2.7 ; os_name == "posix" and python_version < "3"