From 25e77b7c98aebaacfd6ff97e2ff82c8271ca61b5 Mon Sep 17 00:00:00 2001 From: Sloane Hertel <19572925+s-hertel@users.noreply.github.com> Date: Fri, 14 Apr 2023 16:07:11 -0400 Subject: Fetch signatures from galaxy after the dependency resolver runs (#80334) (#80399) Reduce the number of Galaxy API calls made during dependency resolution by fetching remote signatures afterwards, since these are not used in backtracking. Reduce the verbosity to `-vvvv` (to match other Galaxy API calls) to see this activity. Co-authored-by: Sviatoslav Sydorenko (cherry picked from commit 460abe0ceffc5ca99b3cc2f2e3ef07aa2cb225dc) --- .../fragments/80334-reduce-ansible-galaxy-api-calls.yml | 2 ++ lib/ansible/galaxy/api.py | 3 +-- lib/ansible/galaxy/collection/__init__.py | 3 +++ lib/ansible/galaxy/dependency_resolution/dataclasses.py | 12 +++++++++++- lib/ansible/galaxy/dependency_resolution/providers.py | 1 - .../targets/ansible-galaxy-collection/tasks/install.yml | 3 ++- 6 files changed, 19 insertions(+), 5 deletions(-) create mode 100644 changelogs/fragments/80334-reduce-ansible-galaxy-api-calls.yml diff --git a/changelogs/fragments/80334-reduce-ansible-galaxy-api-calls.yml b/changelogs/fragments/80334-reduce-ansible-galaxy-api-calls.yml new file mode 100644 index 0000000000..c780a10961 --- /dev/null +++ b/changelogs/fragments/80334-reduce-ansible-galaxy-api-calls.yml @@ -0,0 +1,2 @@ +bugfixes: + - ansible-galaxy - reduce API calls to servers by fetching signatures only for final candidates. diff --git a/lib/ansible/galaxy/api.py b/lib/ansible/galaxy/api.py index ef16093347..b7c1777f32 100644 --- a/lib/ansible/galaxy/api.py +++ b/lib/ansible/galaxy/api.py @@ -918,8 +918,7 @@ class GalaxyAPI: try: signatures = data["signatures"] except KeyError: - # Noisy since this is used by the dep resolver, so require more verbosity than Galaxy calls - display.vvvvvv(f"Server {self.api_server} has not signed {namespace}.{name}:{version}") + display.vvvv(f"Server {self.api_server} has not signed {namespace}.{name}:{version}") return [] else: return [signature_info["signature"] for signature_info in signatures] diff --git a/lib/ansible/galaxy/collection/__init__.py b/lib/ansible/galaxy/collection/__init__.py index 9812bcade5..c82f2f7259 100644 --- a/lib/ansible/galaxy/collection/__init__.py +++ b/lib/ansible/galaxy/collection/__init__.py @@ -741,6 +741,9 @@ def install_collections( "Skipping signature verification." ) + if concrete_coll_pin.type == 'galaxy': + concrete_coll_pin = concrete_coll_pin.with_signatures_repopulated() + try: install(concrete_coll_pin, output_path, artifacts_manager) except AnsibleError as err: diff --git a/lib/ansible/galaxy/dependency_resolution/dataclasses.py b/lib/ansible/galaxy/dependency_resolution/dataclasses.py index 2ac6892531..0a942d4cce 100644 --- a/lib/ansible/galaxy/dependency_resolution/dataclasses.py +++ b/lib/ansible/galaxy/dependency_resolution/dataclasses.py @@ -27,7 +27,7 @@ if t.TYPE_CHECKING: ) -from ansible.errors import AnsibleError +from ansible.errors import AnsibleError, AnsibleAssertionError from ansible.galaxy.api import GalaxyAPI from ansible.module_utils._text import to_bytes, to_native, to_text from ansible.module_utils.common.arg_spec import ArgumentSpecValidator @@ -571,3 +571,13 @@ class Candidate( def __init__(self, *args, **kwargs): super(Candidate, self).__init__() + + def with_signatures_repopulated(self): # type: (Candidate) -> Candidate + """Populate a new Candidate instance with Galaxy signatures. + :raises AnsibleAssertionError: If the supplied candidate is not sourced from a Galaxy-like index. + """ + if self.type != 'galaxy': + raise AnsibleAssertionError(f"Invalid collection type for {self!r}: unable to get signatures from a galaxy server.") + + signatures = self.src.get_collection_signatures(self.namespace, self.name, self.ver) + return self.__class__(self.fqcn, self.ver, self.src, self.type, frozenset([*self.signatures, *signatures])) diff --git a/lib/ansible/galaxy/dependency_resolution/providers.py b/lib/ansible/galaxy/dependency_resolution/providers.py index 817a1eb227..156fb9462f 100644 --- a/lib/ansible/galaxy/dependency_resolution/providers.py +++ b/lib/ansible/galaxy/dependency_resolution/providers.py @@ -392,7 +392,6 @@ class CollectionDependencyProviderBase(AbstractProvider): if not unsatisfied: if self._include_signatures: - signatures = src_server.get_collection_signatures(first_req.namespace, first_req.name, version) for extra_source in extra_signature_sources: signatures.append(get_signature_from_source(extra_source)) latest_matches.append( diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/install.yml b/test/integration/targets/ansible-galaxy-collection/tasks/install.yml index a55b64d8a9..f9014ed9bd 100644 --- a/test/integration/targets/ansible-galaxy-collection/tasks/install.yml +++ b/test/integration/targets/ansible-galaxy-collection/tasks/install.yml @@ -5,7 +5,7 @@ state: directory - name: install simple collection from first accessible server - command: ansible-galaxy collection install namespace1.name1 {{ galaxy_verbosity }} + command: ansible-galaxy collection install namespace1.name1 -vvvv environment: ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections' register: from_first_good_server @@ -30,6 +30,7 @@ - install_normal_files.files[1].path | basename in ['MANIFEST.json', 'FILES.json', 'README.md'] - install_normal_files.files[2].path | basename in ['MANIFEST.json', 'FILES.json', 'README.md'] - (install_normal_manifest.content | b64decode | from_json).collection_info.version == '1.0.9' + - 'from_first_good_server.stdout|regex_findall("has not signed namespace1\.name1")|length == 1' - name: Remove the collection file: -- cgit v1.2.1