Add TCP keepalive to gitlab

The gitlab driver uses requests to interact with gitlab's API.  By
default, requests uses a connection pool to cache and re-use HTTP
connections. If the gitlab server is across a network segment with,
say, a poor NAT implementation which silently drops idle connections
from its map, then Zuul may not notice until it attempts to make a
request on one of these connections and receives a read timeout.

To avoid this, use TCP keepalives on these HTTP requests by default.

This is configured to send a keepalive packet by default every 60
seconds.  This should avoid sending too much extra traffic, while
improving the reliability for all users.  The default of 60 seconds
was chosen to match the SSH keepalive value we set for gerrit
connections (for much the same reason).

If this works well, we can use this approach with other requests-based
drivers (and at this point, every driver uses requests to at least
some degree).

Change-Id: I86ce064c6ec9325e2ae9db516778058050696ef6

1 files changed, 8 insertions, 3 deletions

diff --git a/zuul/driver/gitlab/gitlabconnection.py b/zuul/driver/gitlab/gitlabconnection.py
index 11edd1094..9a6cbc636 100644
--- a/zuul/driver/gitlab/gitlabconnection.py
+++ b/zuul/driver/gitlab/gitlabconnection.py
@@ -30,6 +30,7 @@ from typing import List, Optional
 
 from zuul.connection import CachedBranchConnection, ZKChangeCacheMixin
 from zuul.web.handler import BaseWebController
+from zuul.lib.http import ZuulHTTPAdapter
 from zuul.lib.logutil import get_annotated_logger
 from zuul.exceptions import MergeFailure
 from zuul.model import Branch, Project, Ref, Tag
@@ -254,11 +255,12 @@ class GitlabAPIClientException(Exception):
 class GitlabAPIClient():
     log = logging.getLogger("zuul.GitlabAPIClient")
 
-    def __init__(self, baseurl, api_token):
+    def __init__(self, baseurl, api_token, keepalive):
         self.session = requests.Session()
         retry = urllib3.util.Retry(total=8,
                                    backoff_factor=0.1)
-        adapter = requests.adapters.HTTPAdapter(max_retries=retry)
+        adapter = ZuulHTTPAdapter(keepalive=keepalive,
+                                  max_retries=retry)
         self.session.mount(baseurl, adapter)
         self.baseurl = '%s/api/v4' % baseurl
         self.api_token = api_token
@@ -435,7 +437,10 @@ class GitlabConnection(ZKChangeCacheMixin, CachedBranchConnection):
             'api_token_name', '')
         self.api_token = self.connection_config.get(
             'api_token', '')
-        self.gl_client = GitlabAPIClient(self.baseurl, self.api_token)
+        self.keepalive = self.connection_config.get('keepalive', 60)
+
+        self.gl_client = GitlabAPIClient(self.baseurl, self.api_token,
+                                         self.keepalive)
         self.sched = None
         self.source = driver.getSource(self)