Support galaxy v3/autohub API in ansible-galaxy (#60982)

* Add galaxy collections API v3 support

Issue: ansible/galaxy-dev#60

- Determine if server supports v3

Use 'available_versions' from `GET /api`
to determine if 'v3' api is available on
the server.

- Support v3 pagination style

ie, 'limit/offset style', with the paginated
responses based on https://jsonapi.org/format/#fetching-pagination

v2 galaxy uses pagination that is more or less
'django rest framework style' or 'page/page_size style',
based on the default drf pagination described
at https://www.django-rest-framework.org/api-guide/pagination/#pagenumberpagination

- Support galaxy v3 style error response

The error objects returned by the galaxy v3 api are based
on the JSONAPI response/errors format
(https://jsonapi.org/format/#errors).

This handles that style response. At least for publish_collection
for now. Needs extracting/generalizing.

Handle HTTPError in CollectionRequirement.from_name()
with _handle_http_error(). It will raise AnsibleError
based on the json in an error response.

- Update unit tests

update test/unit/galaxy/test_collection*
to paramaterize calls to test against
mocked v2 and v3 servers apis.

Update artifacts_versions_json() to tale an
api version paramater.

Add error_json() for generating v3/v3 style error
responses.

So now, the urls generated and the pagination schema
of the response will use the v3 version if
the passed in GalaxyAPI 'galaxy_api' instance
has 'v3' in it's available_api_versions

* Move checking of server avail versions to collections.py

collections.py needs to know the server api versions
supported before it makes collection related calls,
so the 'lazy' server version check in api.GalaxyAPI
is never called and isn't set, so 'v3' servers weren't
found.

Update unit tests to mock the return value of the
request instead of GalaxyAPI itself.

2 files changed, 119 insertions, 23 deletions

diff --git a/lib/ansible/galaxy/api.py b/lib/ansible/galaxy/api.py
index 7df9d3b1af..45de4cbb89 100644
--- a/lib/ansible/galaxy/api.py
+++ b/lib/ansible/galaxy/api.py
@@ -43,11 +43,14 @@ def g_connect(method):
         if not self.initialized:
             display.vvvv("Initial connection to galaxy_server: %s" % self.api_server)
             server_version = self._get_server_api_version()
+
             if server_version not in self.SUPPORTED_VERSIONS:
                 raise AnsibleError("Unsupported Galaxy server API version: %s" % server_version)
 
             self.baseurl = _urljoin(self.api_server, "api", server_version)
+
             self.version = server_version  # for future use
+
             display.vvvv("Base API: %s" % self.baseurl)
             self.initialized = True
         return method(self, *args, **kwargs)
@@ -63,25 +66,32 @@ class GalaxyAPI(object):
 
     SUPPORTED_VERSIONS = ['v1']
 
-    def __init__(self, galaxy, name, url, username=None, password=None, token=None):
+    def __init__(self, galaxy, name, url, username=None, password=None, token=None, token_type=None):
         self.galaxy = galaxy
         self.name = name
         self.username = username
         self.password = password
         self.token = token
+        self.token_type = token_type or 'Token'
         self.api_server = url
         self.validate_certs = not context.CLIARGS['ignore_certs']
         self.baseurl = None
         self.version = None
         self.initialized = False
+        self.available_api_versions = {}
 
         display.debug('Validate TLS certificates for %s: %s' % (self.api_server, self.validate_certs))
 
-    def _auth_header(self, required=True):
+    def _auth_header(self, required=True, token_type=None):
+        '''Generate the Authorization header.
+
+        Valid token_type values are 'Token' (galaxy v2) and 'Bearer' (galaxy v3)'''
         token = self.token.get() if self.token else None
 
+        # 'Token' for v2 api, 'Bearer' for v3
+        token_type = token_type or self.token_type
         if token:
-            return {'Authorization': "Token %s" % token}
+            return {'Authorization': "%s %s" % (token_type, token)}
         elif self.username:
             token = "%s:%s" % (to_text(self.username, errors='surrogate_or_strict'),
                                to_text(self.password, errors='surrogate_or_strict', nonstring='passthru') or '')
@@ -123,9 +133,6 @@ class GalaxyAPI(object):
         except Exception as e:
             raise AnsibleError("Could not process data from the API server (%s): %s " % (url, to_native(e)))
 
-        if 'current_version' not in data:
-            raise AnsibleError("missing required 'current_version' from server response (%s)" % url)
-
         return data['current_version']
 
     @g_connect
diff --git a/lib/ansible/galaxy/collection.py b/lib/ansible/galaxy/collection.py
index 65f2e2e3e4..58c0a25db9 100644
--- a/lib/ansible/galaxy/collection.py
+++ b/lib/ansible/galaxy/collection.py
@@ -310,6 +310,14 @@ class CollectionRequirement:
 
         for api in apis:
             collection_url_paths = [api.api_server, 'api', 'v2', 'collections', namespace, name, 'versions']
+
+            available_api_versions = get_available_api_versions(api)
+            if 'v3' in available_api_versions:
+                # /api/v3/ exists, use it
+                collection_url_paths[2] = 'v3'
+                # update this v3 GalaxyAPI to use Bearer token from now on
+                api.token_type = 'Bearer'
+
             headers = api._auth_header(required=False)
 
             is_single = False
@@ -325,10 +333,13 @@ class CollectionRequirement:
             try:
                 resp = json.load(open_url(n_collection_url, validate_certs=api.validate_certs, headers=headers))
             except urllib_error.HTTPError as err:
+
                 if err.code == 404:
                     display.vvv("Collection '%s' is not available from server %s %s" % (collection, api.name, api.api_server))
                     continue
-                raise
+
+                _handle_http_error(err, api, available_api_versions,
+                                   'Error fetching info for %s from %s (%s)' % (collection, api.name, api.api_server))
 
             if is_single:
                 galaxy_info = resp
@@ -336,13 +347,24 @@ class CollectionRequirement:
                 versions = [resp['version']]
             else:
                 versions = []
+
+                results_key = 'results'
+                if 'v3' in available_api_versions:
+                    results_key = 'data'
+
                 while True:
                     # Galaxy supports semver but ansible-galaxy does not. We ignore any versions that don't match
                     # StrictVersion (x.y.z) and only support pre-releases if an explicit version was set (done above).
-                    versions += [v['version'] for v in resp['results'] if StrictVersion.version_re.match(v['version'])]
-                    if resp['next'] is None:
+                    versions += [v['version'] for v in resp[results_key] if StrictVersion.version_re.match(v['version'])]
+
+                    next_link = resp.get('next', None)
+                    if 'v3' in available_api_versions:
+                        next_link = resp['links']['next']
+
+                    if next_link is None:
                         break
-                    resp = json.load(open_url(to_native(resp['next'], errors='surrogate_or_strict'),
+
+                    resp = json.load(open_url(to_native(next_link, errors='surrogate_or_strict'),
                                               validate_certs=api.validate_certs, headers=headers))
 
             display.vvv("Collection '%s' obtained from server %s %s" % (collection, api.name, api.api_server))
@@ -356,6 +378,45 @@ class CollectionRequirement:
         return req
 
 
+def get_available_api_versions(galaxy_api):
+    headers = {}
+    headers.update(galaxy_api._auth_header(required=False))
+
+    url = _urljoin(galaxy_api.api_server, "api")
+    try:
+        return_data = open_url(url, headers=headers, validate_certs=galaxy_api.validate_certs)
+    except urllib_error.HTTPError as err:
+        if err.code != 401:
+            _handle_http_error(err, galaxy_api, {},
+                               "Error when finding available api versions from %s (%s)" %
+                               (galaxy_api.name, galaxy_api.api_server))
+
+        # assume this is v3 and auth is required.
+        headers = {}
+        headers.update(galaxy_api._auth_header(token_type='Bearer', required=True))
+        # try again with auth
+        try:
+            return_data = open_url(url, headers=headers, validate_certs=galaxy_api.validate_certs)
+        except urllib_error.HTTPError as authed_err:
+            _handle_http_error(authed_err, galaxy_api, {},
+                               "Error when finding available api versions from %s using auth (%s)" %
+                               (galaxy_api.name, galaxy_api.api_server))
+
+    except Exception as e:
+        raise AnsibleError("Failed to get data from the API server (%s): %s " % (url, to_native(e)))
+
+    try:
+        data = json.loads(to_text(return_data.read(), errors='surrogate_or_strict'))
+    except Exception as e:
+        raise AnsibleError("Could not process data from the API server (%s): %s " % (url, to_native(e)))
+
+    available_versions = data.get('available_versions',
+                                  {'v1': '/api/v1',
+                                   'v2': '/api/v2'})
+
+    return available_versions
+
+
 def build_collection(collection_path, output_path, force):
     """
     Creates the Ansible collection artifact in a .tar.gz file.
@@ -410,27 +471,25 @@ def publish_collection(collection_path, api, wait, timeout):
     display.display("Publishing collection artifact '%s' to %s %s" % (collection_path, api.name, api.api_server))
 
     n_url = _urljoin(api.api_server, 'api', 'v2', 'collections')
+    available_api_versions = get_available_api_versions(api)
+
+    if 'v3' in available_api_versions:
+        n_url = _urljoin(api.api_server, 'api', 'v3', 'artifacts', 'collections')
+        api.token_type = 'Bearer'
+
+    headers = {}
+    headers.update(api._auth_header())
 
     data, content_type = _get_mime_data(b_collection_path)
-    headers = {
+    headers.update({
         'Content-type': content_type,
         'Content-length': len(data),
-    }
-    headers.update(api._auth_header())
+    })
 
     try:
         resp = json.load(open_url(n_url, data=data, headers=headers, method='POST', validate_certs=api.validate_certs))
     except urllib_error.HTTPError as err:
-        try:
-            err_info = json.load(err)
-        except (AttributeError, ValueError):
-            err_info = {}
-
-        code = to_native(err_info.get('code', 'Unknown'))
-        message = to_native(err_info.get('message', 'Unknown error returned by Galaxy server.'))
-
-        raise AnsibleError("Error when publishing collection (HTTP Code: %d, Message: %s Code: %s)"
-                           % (err.code, message, code))
+        _handle_http_error(err, api, available_api_versions, "Error when publishing collection to %s (%s)" % (api.name, api.api_server))
 
     import_uri = resp['task']
     if wait:
@@ -1016,3 +1075,33 @@ def _extract_tar_file(tar, filename, b_dest, b_temp_path, expected_hash=None):
             os.makedirs(b_parent_dir)
 
         shutil.move(to_bytes(tmpfile_obj.name, errors='surrogate_or_strict'), b_dest_filepath)
+
+
+def _handle_http_error(http_error, api, available_api_versions, context_error_message):
+    try:
+        err_info = json.load(http_error)
+    except (AttributeError, ValueError):
+        err_info = {}
+
+    if 'v3' in available_api_versions:
+        message_lines = []
+        errors = err_info.get('errors', None)
+
+        if not errors:
+            errors = [{'detail': 'Unknown error returned by Galaxy server.',
+                       'code': 'Unknown'}]
+
+        for error in errors:
+            error_msg = error.get('detail') or error.get('title') or 'Unknown error returned by Galaxy server.'
+            error_code = error.get('code') or 'Unknown'
+            message_line = "(HTTP Code: %d, Message: %s Code: %s)" % (http_error.code, error_msg, error_code)
+            message_lines.append(message_line)
+
+        full_error_msg = "%s %s" % (context_error_message, ', '.join(message_lines))
+    else:
+        code = to_native(err_info.get('code', 'Unknown'))
+        message = to_native(err_info.get('message', 'Unknown error returned by Galaxy server.'))
+        full_error_msg = "%s (HTTP Code: %d, Message: %s Code: %s)" \
+            % (context_error_message, http_error.code, message, code)
+
+    raise AnsibleError(full_error_msg)