Backport ansible-test updates from devel.

The primary change is use of the new servers for
provisioning remote OS X instances for use in CI.

2 files changed, 168 insertions, 32 deletions

diff --git a/test/runner/lib/core_ci.py b/test/runner/lib/core_ci.py
index b7b9ff041e..cd49d86f81 100644
--- a/test/runner/lib/core_ci.py
+++ b/test/runner/lib/core_ci.py
@@ -48,6 +48,8 @@ class AnsibleCoreCI(object):
         self.client = HttpClient(args)
         self.connection = None
         self.instance_id = None
+        self.endpoint = None
+        self.max_threshold = 1
         self.name = name if name else '%s-%s' % (self.platform, self.version)
         self.ci_key = os.path.expanduser('~/.ansible-core-ci.key')
 
@@ -79,7 +81,7 @@ class AnsibleCoreCI(object):
                 # send all non-Shippable jobs to us-east-1 to reduce api key maintenance
                 region = 'us-east-1'
 
-            self.endpoint = AWS_ENDPOINTS[region]
+            self.endpoints = AWS_ENDPOINTS[region],
 
             if self.platform == 'windows':
                 self.ssh_key = None
@@ -88,7 +90,8 @@ class AnsibleCoreCI(object):
                 self.ssh_key = SshKey(args)
                 self.port = 22
         elif self.platform in osx_platforms:
-            self.endpoint = 'https://osx.testing.ansible.com'
+            self.endpoints = self._get_parallels_endpoints()
+            self.max_threshold = 6
 
             self.ssh_key = SshKey(args)
             self.port = None
@@ -115,8 +118,10 @@ class AnsibleCoreCI(object):
                              verbosity=1)
 
                 self.instance_id = None
+                self.endpoint = None
         else:
             self.instance_id = None
+            self.endpoint = None
             self._clear()
 
         if self.instance_id:
@@ -124,23 +129,42 @@ class AnsibleCoreCI(object):
         else:
             self.started = False
             self.instance_id = str(uuid.uuid4())
+            self.endpoint = None
 
-            display.info('Initializing new %s/%s instance %s.' % (self.platform, self.version, self.instance_id),
-                         verbosity=1)
+    def _get_parallels_endpoints(self):
+        """
+        :rtype: tuple[str]
+        """
+        client = HttpClient(self.args, always=True)
+        display.info('Getting available endpoints...', verbosity=1)
+        sleep = 3
+
+        for _ in range(1, 10):
+            response = client.get('https://s3.amazonaws.com/ansible-ci-files/ansible-test/parallels-endpoints.txt')
+
+            if response.status_code == 200:
+                endpoints = tuple(response.response.splitlines())
+                display.info('Available endpoints (%d):\n%s' % (len(endpoints), '\n'.join(' - %s' % endpoint for endpoint in endpoints)), verbosity=1)
+                return endpoints
+
+            display.warning('HTTP %d error getting endpoints, trying again in %d seconds.' % (response.status_code, sleep))
+            time.sleep(sleep)
+
+        raise ApplicationError('Unable to get available endpoints.')
 
     def start(self):
         """Start instance."""
         if is_shippable():
-            self.start_shippable()
-        else:
-            self.start_remote()
+            return self.start_shippable()
+
+        return self.start_remote()
 
     def start_remote(self):
         """Start instance for remote development/testing."""
         with open(self.ci_key, 'r') as key_fd:
             auth_key = key_fd.read().strip()
 
-        self._start(dict(
+        return self._start(dict(
             remote=dict(
                 key=auth_key,
                 nonce=None,
@@ -149,7 +173,7 @@ class AnsibleCoreCI(object):
 
     def start_shippable(self):
         """Start instance on Shippable."""
-        self._start(dict(
+        return self._start(dict(
             shippable=dict(
                 run_id=os.environ['SHIPPABLE_BUILD_ID'],
                 job_number=int(os.environ['SHIPPABLE_JOB_NUMBER']),
@@ -179,7 +203,7 @@ class AnsibleCoreCI(object):
 
         raise self._create_http_error(response)
 
-    def get(self, tries=2, sleep=10, always_raise_on=None):
+    def get(self, tries=3, sleep=15, always_raise_on=None):
         """
         Get instance connection information.
         :type tries: int
@@ -264,9 +288,11 @@ class AnsibleCoreCI(object):
                          verbosity=1)
             return
 
+        display.info('Initializing new %s/%s instance %s.' % (self.platform, self.version, self.instance_id), verbosity=1)
+
         if self.platform == 'windows':
-            with open('examples/scripts/ConfigureRemotingForAnsible.ps1', 'r') as winrm_config_fd:
-                winrm_config = winrm_config_fd.read()
+            with open('examples/scripts/ConfigureRemotingForAnsible.ps1', 'rb') as winrm_config_fd:
+                winrm_config = winrm_config_fd.read().decode('utf-8')
         else:
             winrm_config = None
 
@@ -286,29 +312,76 @@ class AnsibleCoreCI(object):
             'Content-Type': 'application/json',
         }
 
-        tries = 2
-        sleep = 10
+        response = self._start_try_endpoints(data, headers)
+
+        self.started = True
+        self._save()
+
+        display.info('Started %s/%s from: %s' % (self.platform, self.version, self._uri), verbosity=1)
+
+        if self.args.explain:
+            return {}
+
+        return response.json()
+
+    def _start_try_endpoints(self, data, headers):
+        """
+        :type data: dict[str, any]
+        :type headers: dict[str, str]
+        :rtype: HttpResponse
+        """
+        threshold = 1
+
+        while threshold <= self.max_threshold:
+            for self.endpoint in self.endpoints:
+                try:
+                    return self._start_at_threshold(data, headers, threshold)
+                except CoreHttpError as ex:
+                    if ex.status == 503:
+                        display.info('Service Unavailable: %s' % ex.remote_message, verbosity=1)
+                        continue
+                    display.error(ex.remote_message)
+                except HttpError as ex:
+                    display.error(u'%s' % ex)
+
+                time.sleep(3)
+
+            threshold += 1
+
+        raise ApplicationError('Maximum threshold reached and all endpoints exhausted.')
+
+    def _start_at_threshold(self, data, headers, threshold):
+        """
+        :type data: dict[str, any]
+        :type headers: dict[str, str]
+        :type threshold: int
+        :rtype: HttpResponse | None
+        """
+        tries = 3
+        sleep = 15
+
+        data['threshold'] = threshold
+
+        display.info('Trying endpoint: %s (threshold %d)' % (self.endpoint, threshold), verbosity=1)
 
         while True:
             tries -= 1
             response = self.client.put(self._uri, data=json.dumps(data), headers=headers)
 
             if response.status_code == 200:
-                break
+                return response
 
             error = self._create_http_error(response)
 
+            if response.status_code == 503:
+                raise error
+
             if not tries:
                 raise error
 
             display.warning('%s. Trying again after %d seconds.' % (error, sleep))
             time.sleep(sleep)
 
-        self.started = True
-        self._save()
-
-        display.info('Started %s/%s from: %s' % (self.platform, self.version, self._uri), verbosity=1)
-
     def _clear(self):
         """Clear instance information."""
         try:
@@ -322,14 +395,23 @@ class AnsibleCoreCI(object):
         """Load instance information."""
         try:
             with open(self.path, 'r') as instance_fd:
-                self.instance_id = instance_fd.read()
-                self.started = True
+                data = instance_fd.read()
         except IOError as ex:
             if ex.errno != errno.ENOENT:
                 raise
-            self.instance_id = None
 
-        return self.instance_id
+            return False
+
+        if not data.startswith('{'):
+            return False  # legacy format
+
+        config = json.loads(data)
+
+        self.instance_id = config['instance_id']
+        self.endpoint = config['endpoint']
+        self.started = True
+
+        return True
 
     def _save(self):
         """Save instance information."""
@@ -339,7 +421,12 @@ class AnsibleCoreCI(object):
         make_dirs(os.path.dirname(self.path))
 
         with open(self.path, 'w') as instance_fd:
-            instance_fd.write(self.instance_id)
+            config = dict(
+                instance_id=self.instance_id,
+                endpoint=self.endpoint,
+            )
+
+            instance_fd.write(json.dumps(config, indent=4, sort_keys=True))
 
     @staticmethod
     def _create_http_error(response):
@@ -360,7 +447,21 @@ class AnsibleCoreCI(object):
         else:
             message = str(response_json)
 
-        return HttpError(response.status_code, '%s%s' % (message, stack_trace))
+        return CoreHttpError(response.status_code, message, stack_trace)
+
+
+class CoreHttpError(HttpError):
+    """HTTP response as an error."""
+    def __init__(self, status, remote_message, remote_stack_trace):
+        """
+        :type status: int
+        :type remote_message: str
+        :type remote_stack_trace: str
+        """
+        super(CoreHttpError, self).__init__(status, '%s%s' % (remote_message, remote_stack_trace))
+
+        self.remote_message = remote_message
+        self.remote_stack_trace = remote_stack_trace
 
 
 class SshKey(object):
diff --git a/test/runner/lib/http.py b/test/runner/lib/http.py
index 554fdf93d6..e0a3da6a57 100644
--- a/test/runner/lib/http.py
+++ b/test/runner/lib/http.py
@@ -6,17 +6,27 @@ Avoids use of urllib2 due to lack of SNI support.
 from __future__ import absolute_import, print_function
 
 import json
+import time
 
 try:
     from urllib import urlencode
 except ImportError:
-    # noinspection PyCompatibility,PyUnresolvedReferences,PyUnresolvedReferences
+    # noinspection PyCompatibility, PyUnresolvedReferences
     from urllib.parse import urlencode  # pylint: disable=locally-disabled, import-error, no-name-in-module
 
+try:
+    # noinspection PyCompatibility
+    from urlparse import urlparse, urlunparse, parse_qs
+except ImportError:
+    # noinspection PyCompatibility, PyUnresolvedReferences
+    from urllib.parse import urlparse, urlunparse, parse_qs  # pylint: disable=locally-disabled, ungrouped-imports
+
 from lib.util import (
     CommonConfig,
     ApplicationError,
     run_command,
+    SubprocessError,
+    display,
 )
 
 
@@ -76,10 +86,31 @@ class HttpClient(object):
 
         cmd += [url]
 
-        stdout, _ = run_command(self.args, cmd, capture=True, always=self.always, cmd_verbosity=2)
+        attempts = 0
+        max_attempts = 3
+        sleep_seconds = 3
+
+        # curl error codes which are safe to retry (request never sent to server)
+        retry_on_status = (
+            6,  # CURLE_COULDNT_RESOLVE_HOST
+        )
+
+        while True:
+            attempts += 1
+
+            try:
+                stdout, _ = run_command(self.args, cmd, capture=True, always=self.always, cmd_verbosity=2)
+                break
+            except SubprocessError as ex:
+                if ex.status in retry_on_status and attempts < max_attempts:
+                    display.warning(u'%s' % ex)
+                    time.sleep(sleep_seconds)
+                    continue
+
+                raise
 
         if self.args.explain and not self.always:
-            return HttpResponse(200, '')
+            return HttpResponse(method, url, 200, '')
 
         header, body = stdout.split('\r\n\r\n', 1)
 
@@ -88,16 +119,20 @@ class HttpClient(object):
         http_response = first_line.split(' ')
         status_code = int(http_response[1])
 
-        return HttpResponse(status_code, body)
+        return HttpResponse(method, url, status_code, body)
 
 
 class HttpResponse(object):
     """HTTP response from curl."""
-    def __init__(self, status_code, response):
+    def __init__(self, method, url, status_code, response):
         """
+        :type method: str
+        :type url: str
         :type status_code: int
         :type response: str
         """
+        self.method = method
+        self.url = url
         self.status_code = status_code
         self.response = response
 
@@ -108,7 +143,7 @@ class HttpResponse(object):
         try:
             return json.loads(self.response)
         except ValueError:
-            raise HttpError(self.status_code, 'Cannot parse response as JSON:\n%s' % self.response)
+            raise HttpError(self.status_code, 'Cannot parse response to %s %s as JSON:\n%s' % (self.method, self.url, self.response))
 
 
 class HttpError(ApplicationError):