diff options
| -rw-r--r-- | changelogs/fragments/78325-ansible-galaxy-fix-caching-paginated-responses-from-v3-servers.yml | 2 | ||||
| -rw-r--r-- | lib/ansible/galaxy/api.py | 23 | ||||
| -rw-r--r-- | test/units/galaxy/test_api.py | 80 |
3 files changed, 95 insertions, 10 deletions
diff --git a/changelogs/fragments/78325-ansible-galaxy-fix-caching-paginated-responses-from-v3-servers.yml b/changelogs/fragments/78325-ansible-galaxy-fix-caching-paginated-responses-from-v3-servers.yml new file mode 100644 index 0000000000..c5cf55e3cd --- /dev/null +++ b/changelogs/fragments/78325-ansible-galaxy-fix-caching-paginated-responses-from-v3-servers.yml @@ -0,0 +1,2 @@ +bugfixes: + - ansible-galaxy - fix setting the cache for paginated responses from Galaxy NG/AH (https://github.com/ansible/ansible/issues/77911). diff --git a/lib/ansible/galaxy/api.py b/lib/ansible/galaxy/api.py index c09acc2dbb..8dea804953 100644 --- a/lib/ansible/galaxy/api.py +++ b/lib/ansible/galaxy/api.py @@ -330,25 +330,27 @@ class GalaxyAPI: should_retry_error=is_rate_limit_exception ) def _call_galaxy(self, url, args=None, headers=None, method=None, auth_required=False, error_context_msg=None, - cache=False): + cache=False, cache_key=None): url_info = urlparse(url) cache_id = get_cache_id(url) + if not cache_key: + cache_key = url_info.path query = parse_qs(url_info.query) if cache and self._cache: server_cache = self._cache.setdefault(cache_id, {}) iso_datetime_format = '%Y-%m-%dT%H:%M:%SZ' valid = False - if url_info.path in server_cache: - expires = datetime.datetime.strptime(server_cache[url_info.path]['expires'], iso_datetime_format) + if cache_key in server_cache: + expires = datetime.datetime.strptime(server_cache[cache_key]['expires'], iso_datetime_format) valid = datetime.datetime.utcnow() < expires is_paginated_url = 'page' in query or 'offset' in query if valid and not is_paginated_url: # Got a hit on the cache and we aren't getting a paginated response - path_cache = server_cache[url_info.path] + path_cache = server_cache[cache_key] if path_cache.get('paginated'): - if '/v3/' in url_info.path: + if '/v3/' in cache_key: res = {'links': {'next': None}} else: res = {'next': None} @@ -368,7 +370,7 @@ class GalaxyAPI: # The cache entry had expired or does not exist, start a new blank entry to be filled later. expires = datetime.datetime.utcnow() expires += datetime.timedelta(days=1) - server_cache[url_info.path] = { + server_cache[cache_key] = { 'expires': expires.strftime(iso_datetime_format), 'paginated': False, } @@ -393,7 +395,7 @@ class GalaxyAPI: % (resp.url, to_native(resp_data))) if cache and self._cache: - path_cache = self._cache[cache_id][url_info.path] + path_cache = self._cache[cache_id][cache_key] # v3 can return data or results for paginated results. Scan the result so we can determine what to cache. paginated_key = None @@ -815,6 +817,7 @@ class GalaxyAPI: page_size_name = 'limit' if 'v3' in self.available_api_versions else 'page_size' versions_url = _urljoin(self.api_server, api_path, 'collections', namespace, name, 'versions', '/?%s=%d' % (page_size_name, COLLECTION_PAGE_SIZE)) versions_url_info = urlparse(versions_url) + cache_key = versions_url_info.path # We should only rely on the cache if the collection has not changed. This may slow things down but it ensures # we are not waiting a day before finding any new collections that have been published. @@ -834,7 +837,7 @@ class GalaxyAPI: if cached_modified_date != modified_date: modified_cache['%s.%s' % (namespace, name)] = modified_date if versions_url_info.path in server_cache: - del server_cache[versions_url_info.path] + del server_cache[cache_key] self._set_cache() @@ -842,7 +845,7 @@ class GalaxyAPI: % (namespace, name, self.name, self.api_server) try: - data = self._call_galaxy(versions_url, error_context_msg=error_context_msg, cache=True) + data = self._call_galaxy(versions_url, error_context_msg=error_context_msg, cache=True, cache_key=cache_key) except GalaxyError as err: if err.http_code != 404: raise @@ -876,7 +879,7 @@ class GalaxyAPI: next_link = versions_url.replace(versions_url_info.path, next_link) data = self._call_galaxy(to_native(next_link, errors='surrogate_or_strict'), - error_context_msg=error_context_msg, cache=True) + error_context_msg=error_context_msg, cache=True, cache_key=cache_key) self._set_cache() return versions diff --git a/test/units/galaxy/test_api.py b/test/units/galaxy/test_api.py index 1e291555ae..064aff2955 100644 --- a/test/units/galaxy/test_api.py +++ b/test/units/galaxy/test_api.py @@ -75,6 +75,57 @@ def get_test_galaxy_api(url, version, token_ins=None, token_value=None, no_cache return api +def get_v3_collection_versions(namespace='namespace', name='collection'): + pagination_path = f"/api/galaxy/content/community/v3/plugin/{namespace}/content/community/collections/index/{namespace}/{name}/versions" + page_versions = (('1.0.0', '1.0.1',), ('1.0.2', '1.0.3',), ('1.0.4', '1.0.5'),) + responses = [ + {}, # TODO: initial response + ] + + first = f"{pagination_path}/?limit=100" + last = f"{pagination_path}/?limit=100&offset=200" + page_versions = [ + { + "versions": ('1.0.0', '1.0.1',), + "url": first, + }, + { + "versions": ('1.0.2', '1.0.3',), + "url": f"{pagination_path}/?limit=100&offset=100", + }, + { + "versions": ('1.0.4', '1.0.5'), + "url": last, + }, + ] + + previous = None + for page in range(0, len(page_versions)): + data = [] + + if page_versions[page]["url"] == last: + next_page = None + else: + next_page = page_versions[page + 1]["url"] + links = {"first": first, "last": last, "next": next_page, "previous": previous} + + for version in page_versions[page]["versions"]: + data.append( + { + "version": f"{version}", + "href": f"{pagination_path}/{version}/", + "created_at": "2022-05-13T15:55:58.913107Z", + "updated_at": "2022-05-13T15:55:58.913121Z", + "requires_ansible": ">=2.9.10" + } + ) + + responses.append({"meta": {"count": 6}, "links": links, "data": data}) + + previous = page_versions[page]["url"] + return responses + + def get_collection_versions(namespace='namespace', name='collection'): base_url = 'https://galaxy.server.com/api/v2/collections/{0}/{1}/'.format(namespace, name) versions_url = base_url + 'versions/' @@ -1149,6 +1200,35 @@ def test_cache_complete_pagination(cache_dir, monkeypatch): assert cached_versions == actual_versions +def test_cache_complete_pagination_v3(cache_dir, monkeypatch): + + responses = get_v3_collection_versions() + cache_file = os.path.join(cache_dir, 'api.json') + + api = get_test_galaxy_api('https://galaxy.server.com/api/', 'v3', no_cache=False) + + mock_open = MagicMock( + side_effect=[ + StringIO(to_text(json.dumps(r))) + for r in responses + ] + ) + monkeypatch.setattr(galaxy_api, 'open_url', mock_open) + + actual_versions = api.get_collection_versions('namespace', 'collection') + assert actual_versions == [u'1.0.0', u'1.0.1', u'1.0.2', u'1.0.3', u'1.0.4', u'1.0.5'] + + with open(cache_file) as fd: + final_cache = json.loads(fd.read()) + + cached_server = final_cache['galaxy.server.com:'] + cached_collection = cached_server['/api/v3/collections/namespace/collection/versions/'] + cached_versions = [r['version'] for r in cached_collection['results']] + + assert final_cache == api._cache + assert cached_versions == actual_versions + + def test_cache_flaky_pagination(cache_dir, monkeypatch): responses = get_collection_versions() |