Try all galaxy servers when locating available versions for a collection (#75468) (#75750)

* If an exception occurs when getting a collection's metadata, continue to the next in the server list.

* Warn for unknown exceptions when finding versions of a collection

* Test that an invalid server is no longer fatal if a subsequent server has the collection

* Fix server for verify tests - compare checksums against the server from which it was installed

* Add tests for verify and fix that code path to mirror install/download behavior for server errors

Co-authored-by: Sviatoslav Sydorenko <wk.cvs.github@sydorenko.org.ua>
(cherry picked from commit 469b559ebe44451ad60b2e534e284b19090b1c18)

6 files changed, 133 insertions, 16 deletions

diff --git a/changelogs/fragments/75468_fix_galaxy_server_fallback.yaml b/changelogs/fragments/75468_fix_galaxy_server_fallback.yaml
new file mode 100644
index 0000000000..e7448f1435
--- /dev/null
+++ b/changelogs/fragments/75468_fix_galaxy_server_fallback.yaml
@@ -0,0 +1,9 @@
+bugfixes:
+  - >-
+    ansible-galaxy - Fix handling HTTP exceptions from Galaxy servers.
+    Continue to the next server in the list until the collection is found.
+minor_changes:
+  - >-
+    ansible-galaxy - Non-HTTP exceptions from Galaxy servers are now a warning and only fatal if
+    the collection to download|install|verify is not available from any of the servers
+    (https://github.com/ansible/ansible/issues/75443).
diff --git a/lib/ansible/galaxy/collection/galaxy_api_proxy.py b/lib/ansible/galaxy/collection/galaxy_api_proxy.py
index fb4cd5de02..9359375bda 100644
--- a/lib/ansible/galaxy/collection/galaxy_api_proxy.py
+++ b/lib/ansible/galaxy/collection/galaxy_api_proxy.py
@@ -24,6 +24,11 @@ if TYPE_CHECKING:
     )
 
 from ansible.galaxy.api import GalaxyAPI, GalaxyError
+from ansible.module_utils._text import to_text
+from ansible.utils.display import Display
+
+
+display = Display()
 
 
 class MultiGalaxyAPIProxy:
@@ -35,6 +40,47 @@ class MultiGalaxyAPIProxy:
         self._apis = apis
         self._concrete_art_mgr = concrete_artifacts_manager
 
+    def _get_collection_versions(self, requirement):
+        # type: (Requirement, Iterator[GalaxyAPI]) -> Iterator[Tuple[GalaxyAPI, str]]
+        """Helper for get_collection_versions.
+
+        Yield api, version pairs for all APIs,
+        and reraise the last error if no valid API was found.
+        """
+        found_api = False
+        last_error = None
+
+        api_lookup_order = (
+            (requirement.src, )
+            if isinstance(requirement.src, GalaxyAPI)
+            else self._apis
+        )
+
+        for api in api_lookup_order:
+            try:
+                versions = api.get_collection_versions(requirement.namespace, requirement.name)
+            except GalaxyError as api_err:
+                last_error = api_err
+            except Exception as unknown_err:
+                display.warning(
+                    "Skipping Galaxy server {server!s}. "
+                    "Got an unexpected error when getting "
+                    "available versions of collection {fqcn!s}: {err!s}".
+                    format(
+                        server=api.api_server,
+                        fqcn=requirement.fqcn,
+                        err=to_text(unknown_err),
+                    )
+                )
+                last_error = unknown_err
+            else:
+                found_api = True
+                for version in versions:
+                    yield api, version
+
+        if not found_api and last_error is not None:
+            raise last_error
+
     def get_collection_versions(self, requirement):
         # type: (Requirement) -> Iterable[Tuple[str, GalaxyAPI]]
         """Get a set of unique versions for FQCN on Galaxy servers."""
@@ -54,9 +100,8 @@ class MultiGalaxyAPIProxy:
         )
         return set(
             (version, api)
-            for api in api_lookup_order
-            for version in api.get_collection_versions(
-                requirement.namespace, requirement.name,
+            for api, version in self._get_collection_versions(
+                requirement,
             )
         )
 
@@ -78,6 +123,21 @@ class MultiGalaxyAPIProxy:
                 )
             except GalaxyError as api_err:
                 last_err = api_err
+            except Exception as unknown_err:
+                # `verify` doesn't use `get_collection_versions` since the version is already known.
+                # Do the same as `install` and `download` by trying all APIs before failing.
+                # Warn for debugging purposes, since the Galaxy server may be unexpectedly down.
+                last_err = unknown_err
+                display.warning(
+                    "Skipping Galaxy server {server!s}. "
+                    "Got an unexpected error when getting "
+                    "available versions of collection {fqcn!s}: {err!s}".
+                    format(
+                        server=api.api_server,
+                        fqcn=collection_candidate.fqcn,
+                        err=to_text(unknown_err),
+                    )
+                )
             else:
                 self._concrete_art_mgr.save_collection_source(
                     collection_candidate,
diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/install.yml b/test/integration/targets/ansible-galaxy-collection/tasks/install.yml
index 66d79e5947..7d66be2f8a 100644
--- a/test/integration/targets/ansible-galaxy-collection/tasks/install.yml
+++ b/test/integration/targets/ansible-galaxy-collection/tasks/install.yml
@@ -4,6 +4,38 @@
     path: '{{ galaxy_dir }}/ansible_collections'
     state: directory
 
+- name: install simple collection from first accessible server
+  command: ansible-galaxy collection install namespace1.name1 {{ galaxy_verbosity }}
+  environment:
+    ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections'
+  register: from_first_good_server
+
+- name: get installed files of install simple collection from first good server
+  find:
+    path: '{{ galaxy_dir }}/ansible_collections/namespace1/name1'
+    file_type: file
+  register: install_normal_files
+
+- name: get the manifest of install simple collection from first good server
+  slurp:
+    path: '{{ galaxy_dir }}/ansible_collections/namespace1/name1/MANIFEST.json'
+  register: install_normal_manifest
+
+- name: assert install simple collection from first good server
+  assert:
+    that:
+    - '"Installing ''namespace1.name1:1.0.9'' to" in from_first_good_server.stdout'
+    - install_normal_files.files | length == 3
+    - install_normal_files.files[0].path | basename in ['MANIFEST.json', 'FILES.json', 'README.md']
+    - install_normal_files.files[1].path | basename in ['MANIFEST.json', 'FILES.json', 'README.md']
+    - install_normal_files.files[2].path | basename in ['MANIFEST.json', 'FILES.json', 'README.md']
+    - (install_normal_manifest.content | b64decode | from_json).collection_info.version == '1.0.9'
+
+- name: Remove the collection
+  file:
+    path: '{{ galaxy_dir }}/ansible_collections/namespace1'
+    state: absent
+
 - name: install simple collection with implicit path - {{ test_name }}
   command: ansible-galaxy collection install namespace1.name1 -s '{{ test_name }}' {{ galaxy_verbosity }}
   environment:
diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/main.yml b/test/integration/targets/ansible-galaxy-collection/tasks/main.yml
index a536a720d6..0f6af191da 100644
--- a/test/integration/targets/ansible-galaxy-collection/tasks/main.yml
+++ b/test/integration/targets/ansible-galaxy-collection/tasks/main.yml
@@ -176,9 +176,10 @@
       environment:
         ANSIBLE_CONFIG: '{{ galaxy_dir }}/ansible.cfg'
   vars:
+    test_api_fallback: 'pulp_v2'
+    test_api_fallback_versions: 'v1, v2'
     test_name: 'galaxy_ng'
     test_server: '{{ galaxy_ng_server }}'
-    vX: "v3/"
 
 - name: run ansible-galaxy collection list tests
   include_tasks: list.yml
diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/verify.yml b/test/integration/targets/ansible-galaxy-collection/tasks/verify.yml
index eacb8d6b30..8bf3957703 100644
--- a/test/integration/targets/ansible-galaxy-collection/tasks/verify.yml
+++ b/test/integration/targets/ansible-galaxy-collection/tasks/verify.yml
@@ -25,12 +25,24 @@
         "ERROR! 'file' type is not supported. The format namespace.name is expected." in verify.stderr
 
 - name: install the collection from the server
-  command: ansible-galaxy collection install ansible_test.verify:1.0.0
+  command: ansible-galaxy collection install ansible_test.verify:1.0.0 -s {{ test_api_fallback }} {{ galaxy_verbosity }}
   environment:
     ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}'
 
+- name: verify the collection against the first valid server
+  command: ansible-galaxy collection verify ansible_test.verify:1.0.0 -vvv {{ galaxy_verbosity }}
+  environment:
+    ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}'
+  register: verify
+
+- assert:
+    that:
+      - verify is success
+      - >-
+        "Found API version '{{ test_api_fallback_versions }}' with Galaxy server {{ test_api_fallback }}" in verify.stdout
+
 - name: verify the installed collection against the server
-  command: ansible-galaxy collection verify ansible_test.verify:1.0.0
+  command: ansible-galaxy collection verify ansible_test.verify:1.0.0 -s {{ test_name }} {{ galaxy_verbosity }}
   environment:
     ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}'
   register: verify
@@ -41,12 +53,12 @@
       - "'Collection ansible_test.verify contains modified content' not in verify.stdout"
 
 - name: verify the installed collection against the server, with unspecified version in CLI
-  command: ansible-galaxy collection verify ansible_test.verify
+  command: ansible-galaxy collection verify ansible_test.verify -s {{ test_name }} {{ galaxy_verbosity }}
   environment:
     ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}'
 
 - name: verify a collection that doesn't appear to be installed
-  command: ansible-galaxy collection verify ansible_test.verify:1.0.0
+  command: ansible-galaxy collection verify ansible_test.verify:1.0.0 -s {{ test_name }} {{ galaxy_verbosity }}
   register: verify
   failed_when: verify.rc == 0
 
@@ -82,7 +94,7 @@
     chdir: '{{ galaxy_dir }}'
 
 - name: verify a version of a collection that isn't installed
-  command: ansible-galaxy collection verify ansible_test.verify:2.0.0
+  command: ansible-galaxy collection verify ansible_test.verify:2.0.0 -s {{ test_name }} {{ galaxy_verbosity }}
   environment:
     ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}'
   register: verify
@@ -94,12 +106,12 @@
       - '"ansible_test.verify has the version ''1.0.0'' but is being compared to ''2.0.0''" in verify.stdout'
 
 - name: install the new version from the server
-  command: ansible-galaxy collection install ansible_test.verify:2.0.0 --force
+  command: ansible-galaxy collection install ansible_test.verify:2.0.0 --force -s {{ test_name }} {{ galaxy_verbosity }}
   environment:
     ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}'
 
 - name: verify the installed collection against the server
-  command: ansible-galaxy collection verify ansible_test.verify:2.0.0
+  command: ansible-galaxy collection verify ansible_test.verify:2.0.0 -s {{ test_name }} {{ galaxy_verbosity }}
   environment:
     ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}'
   register: verify
@@ -145,7 +157,7 @@
       - "updated_file.stat.checksum != file.stat.checksum"
 
 - name: test verifying checksumes of the modified collection
-  command: ansible-galaxy collection verify ansible_test.verify:2.0.0
+  command: ansible-galaxy collection verify ansible_test.verify:2.0.0 -s {{ test_name }} {{ galaxy_verbosity }}
   register: verify
   environment:
     ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}'
@@ -165,7 +177,7 @@
   diff: true
 
 - name: ensure a modified FILES.json is validated
-  command: ansible-galaxy collection verify ansible_test.verify:2.0.0
+  command: ansible-galaxy collection verify ansible_test.verify:2.0.0 -s {{ test_name }} {{ galaxy_verbosity }}
   register: verify
   environment:
     ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}'
@@ -189,7 +201,7 @@
     line: ' "chksum_sha256": "{{ manifest_info.stat.checksum }}",'
 
 - name: ensure the MANIFEST.json is validated against the uncorrupted file from the server
-  command: ansible-galaxy collection verify ansible_test.verify:2.0.0
+  command: ansible-galaxy collection verify ansible_test.verify:2.0.0 -s {{ test_name }} {{ galaxy_verbosity }}
   register: verify
   environment:
     ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}'
@@ -219,7 +231,7 @@
     dest: '{{ galaxy_dir }}/ansible_collections/ansible_test/verify/galaxy.yml'
 
 - name: test we only verify collections containing a MANIFEST.json with the version on the server
-  command: ansible-galaxy collection verify ansible_test.verify:2.0.0
+  command: ansible-galaxy collection verify ansible_test.verify:2.0.0 -s {{ test_name }} {{ galaxy_verbosity }}
   register: verify
   environment:
     ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}'
diff --git a/test/integration/targets/ansible-galaxy-collection/templates/ansible.cfg.j2 b/test/integration/targets/ansible-galaxy-collection/templates/ansible.cfg.j2
index 62f3dcf901..00c1fe4deb 100644
--- a/test/integration/targets/ansible-galaxy-collection/templates/ansible.cfg.j2
+++ b/test/integration/targets/ansible-galaxy-collection/templates/ansible.cfg.j2
@@ -1,7 +1,10 @@
 [galaxy]
 # Ensures subsequent unstable reruns don't use the cached information causing another failure
 cache_dir={{ remote_tmp_dir }}/galaxy_cache
-server_list=pulp_v2,pulp_v3,galaxy_ng,secondary
+server_list=offline,pulp_v2,pulp_v3,galaxy_ng,secondary
+
+[galaxy_server.offline]
+url=https://test-hub.demolab.local/api/galaxy/content/api/
 
 [galaxy_server.pulp_v2]
 url={{ pulp_server }}published/api/