diff options
Diffstat (limited to 'lib/ansible/galaxy')
-rw-r--r-- | lib/ansible/galaxy/api.py | 36 | ||||
-rw-r--r-- | lib/ansible/galaxy/collection/concrete_artifact_manager.py | 51 |
2 files changed, 62 insertions, 25 deletions
diff --git a/lib/ansible/galaxy/api.py b/lib/ansible/galaxy/api.py index 437b3a26ce..0d51998044 100644 --- a/lib/ansible/galaxy/api.py +++ b/lib/ansible/galaxy/api.py @@ -11,12 +11,15 @@ import functools import hashlib import json import os +import socket import stat import tarfile import time import threading -from urllib.error import HTTPError +from http import HTTPStatus +from http.client import BadStatusLine, IncompleteRead +from urllib.error import HTTPError, URLError from urllib.parse import quote as urlquote, urlencode, urlparse, parse_qs, urljoin from ansible import constants as C @@ -34,10 +37,11 @@ from ansible.utils.path import makedirs_safe display = Display() _CACHE_LOCK = threading.Lock() COLLECTION_PAGE_SIZE = 100 -RETRY_HTTP_ERROR_CODES = [ # TODO: Allow user-configuration - 429, # Too Many Requests +RETRY_HTTP_ERROR_CODES = { # TODO: Allow user-configuration + HTTPStatus.TOO_MANY_REQUESTS, 520, # Galaxy rate limit error code (Cloudflare unknown error) -] + HTTPStatus.BAD_GATEWAY, # Common error from galaxy that may represent any number of transient backend issues +} def cache_lock(func): @@ -48,11 +52,24 @@ def cache_lock(func): return wrapped -def is_rate_limit_exception(exception): +def should_retry_error(exception): # Note: cloud.redhat.com masks rate limit errors with 403 (Forbidden) error codes. # Since 403 could reflect the actual problem (such as an expired token), we should # not retry by default. - return isinstance(exception, GalaxyError) and exception.http_code in RETRY_HTTP_ERROR_CODES + if isinstance(exception, GalaxyError) and exception.http_code in RETRY_HTTP_ERROR_CODES: + return True + + if isinstance(exception, AnsibleError) and (orig_exc := getattr(exception, 'orig_exc', None)): + # URLError is often a proxy for an underlying error, handle wrapped exceptions + if isinstance(orig_exc, URLError): + orig_exc = orig_exc.reason + + # Handle common URL related errors such as TimeoutError, and BadStatusLine + # Note: socket.timeout is only required for Py3.9 + if isinstance(orig_exc, (TimeoutError, BadStatusLine, IncompleteRead, socket.timeout)): + return True + + return False def g_connect(versions): @@ -327,7 +344,7 @@ class GalaxyAPI: @retry_with_delays_and_condition( backoff_iterator=generate_jittered_backoff(retries=6, delay_base=2, delay_threshold=40), - should_retry_error=is_rate_limit_exception + should_retry_error=should_retry_error ) def _call_galaxy(self, url, args=None, headers=None, method=None, auth_required=False, error_context_msg=None, cache=False, cache_key=None): @@ -385,7 +402,10 @@ class GalaxyAPI: except HTTPError as e: raise GalaxyError(e, error_context_msg) except Exception as e: - raise AnsibleError("Unknown error when attempting to call Galaxy at '%s': %s" % (url, to_native(e))) + raise AnsibleError( + "Unknown error when attempting to call Galaxy at '%s': %s" % (url, to_native(e)), + orig_exc=e + ) resp_data = to_text(resp.read(), errors='surrogate_or_strict') try: diff --git a/lib/ansible/galaxy/collection/concrete_artifact_manager.py b/lib/ansible/galaxy/collection/concrete_artifact_manager.py index 7c920b85dd..67d8e43f6a 100644 --- a/lib/ansible/galaxy/collection/concrete_artifact_manager.py +++ b/lib/ansible/galaxy/collection/concrete_artifact_manager.py @@ -27,9 +27,12 @@ if t.TYPE_CHECKING: from ansible.errors import AnsibleError from ansible.galaxy import get_collections_galaxy_meta_info +from ansible.galaxy.api import should_retry_error from ansible.galaxy.dependency_resolution.dataclasses import _GALAXY_YAML from ansible.galaxy.user_agent import user_agent from ansible.module_utils._text import to_bytes, to_native, to_text +from ansible.module_utils.api import retry_with_delays_and_condition +from ansible.module_utils.api import generate_jittered_backoff from ansible.module_utils.common.process import get_bin_path from ansible.module_utils.common._collections_compat import MutableMapping from ansible.module_utils.common.yaml import yaml_load @@ -160,17 +163,24 @@ class ConcreteArtifactsManager: token=token, ) # type: bytes except URLError as err: - raise_from( - AnsibleError( - 'Failed to download collection tar ' - "from '{coll_src!s}': {download_err!s}". - format( - coll_src=to_native(collection.src), - download_err=to_native(err), - ), + raise AnsibleError( + 'Failed to download collection tar ' + "from '{coll_src!s}': {download_err!s}". + format( + coll_src=to_native(collection.src), + download_err=to_native(err), ), - err, - ) + ) from err + except Exception as err: + raise AnsibleError( + 'Failed to download collection tar ' + "from '{coll_src!s}' due to the following unforeseen error: " + '{download_err!s}'. + format( + coll_src=to_native(collection.src), + download_err=to_native(err), + ), + ) from err else: display.vvv( "Collection '{coll!s}' obtained from " @@ -460,6 +470,10 @@ def _extract_collection_from_git(repo_url, coll_ver, b_path): # FIXME: use random subdirs while preserving the file names +@retry_with_delays_and_condition( + backoff_iterator=generate_jittered_backoff(retries=6, delay_base=2, delay_threshold=40), + should_retry_error=should_retry_error +) def _download_file(url, b_path, expected_hash, validate_certs, token=None, timeout=60): # type: (str, bytes, t.Optional[str], bool, GalaxyToken, int) -> bytes # ^ NOTE: used in download and verify_collections ^ @@ -478,13 +492,16 @@ def _download_file(url, b_path, expected_hash, validate_certs, token=None, timeo display.display("Downloading %s to %s" % (url, to_text(b_tarball_dir))) # NOTE: Galaxy redirects downloads to S3 which rejects the request # NOTE: if an Authorization header is attached so don't redirect it - resp = open_url( - to_native(url, errors='surrogate_or_strict'), - validate_certs=validate_certs, - headers=None if token is None else token.headers(), - unredirected_headers=['Authorization'], http_agent=user_agent(), - timeout=timeout - ) + try: + resp = open_url( + to_native(url, errors='surrogate_or_strict'), + validate_certs=validate_certs, + headers=None if token is None else token.headers(), + unredirected_headers=['Authorization'], http_agent=user_agent(), + timeout=timeout + ) + except Exception as err: + raise AnsibleError(to_native(err), orig_exc=err) with open(b_file_path, 'wb') as download_file: # type: t.BinaryIO actual_hash = _consume_file(resp, write_to=download_file) |