Try all galaxy servers when locating available versions for a collection (#75468)

* If an exception occurs when getting a collection's metadata, continue to the next in the server list.

* Warn for unknown exceptions when finding versions of a collection

* Test that an invalid server is no longer fatal if a subsequent server has the collection

* Fix server for verify tests - compare checksums against the server from which it was installed

* Add tests for verify and fix that code path to mirror install/download behavior for server errors

Co-authored-by: Sviatoslav Sydorenko <wk.cvs.github@sydorenko.org.ua>

4 files changed, 61 insertions, 13 deletions

diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/install.yml b/test/integration/targets/ansible-galaxy-collection/tasks/install.yml
index 8b25039a7b..ad10bff87d 100644
--- a/test/integration/targets/ansible-galaxy-collection/tasks/install.yml
+++ b/test/integration/targets/ansible-galaxy-collection/tasks/install.yml
@@ -4,6 +4,38 @@
     path: '{{ galaxy_dir }}/ansible_collections'
     state: directory
 
+- name: install simple collection from first accessible server
+  command: ansible-galaxy collection install namespace1.name1 {{ galaxy_verbosity }}
+  environment:
+    ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections'
+  register: from_first_good_server
+
+- name: get installed files of install simple collection from first good server
+  find:
+    path: '{{ galaxy_dir }}/ansible_collections/namespace1/name1'
+    file_type: file
+  register: install_normal_files
+
+- name: get the manifest of install simple collection from first good server
+  slurp:
+    path: '{{ galaxy_dir }}/ansible_collections/namespace1/name1/MANIFEST.json'
+  register: install_normal_manifest
+
+- name: assert install simple collection from first good server
+  assert:
+    that:
+    - '"Installing ''namespace1.name1:1.0.9'' to" in from_first_good_server.stdout'
+    - install_normal_files.files | length == 3
+    - install_normal_files.files[0].path | basename in ['MANIFEST.json', 'FILES.json', 'README.md']
+    - install_normal_files.files[1].path | basename in ['MANIFEST.json', 'FILES.json', 'README.md']
+    - install_normal_files.files[2].path | basename in ['MANIFEST.json', 'FILES.json', 'README.md']
+    - (install_normal_manifest.content | b64decode | from_json).collection_info.version == '1.0.9'
+
+- name: Remove the collection
+  file:
+    path: '{{ galaxy_dir }}/ansible_collections/namespace1'
+    state: absent
+
 - name: install simple collection with implicit path - {{ test_name }}
   command: ansible-galaxy collection install namespace1.name1 -s '{{ test_name }}' {{ galaxy_verbosity }}
   environment:
diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/main.yml b/test/integration/targets/ansible-galaxy-collection/tasks/main.yml
index a536a720d6..0f6af191da 100644
--- a/test/integration/targets/ansible-galaxy-collection/tasks/main.yml
+++ b/test/integration/targets/ansible-galaxy-collection/tasks/main.yml
@@ -176,9 +176,10 @@
       environment:
         ANSIBLE_CONFIG: '{{ galaxy_dir }}/ansible.cfg'
   vars:
+    test_api_fallback: 'pulp_v2'
+    test_api_fallback_versions: 'v1, v2'
     test_name: 'galaxy_ng'
     test_server: '{{ galaxy_ng_server }}'
-    vX: "v3/"
 
 - name: run ansible-galaxy collection list tests
   include_tasks: list.yml
diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/verify.yml b/test/integration/targets/ansible-galaxy-collection/tasks/verify.yml
index eacb8d6b30..8bf3957703 100644
--- a/test/integration/targets/ansible-galaxy-collection/tasks/verify.yml
+++ b/test/integration/targets/ansible-galaxy-collection/tasks/verify.yml
@@ -25,12 +25,24 @@
         "ERROR! 'file' type is not supported. The format namespace.name is expected." in verify.stderr
 
 - name: install the collection from the server
-  command: ansible-galaxy collection install ansible_test.verify:1.0.0
+  command: ansible-galaxy collection install ansible_test.verify:1.0.0 -s {{ test_api_fallback }} {{ galaxy_verbosity }}
   environment:
     ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}'
 
+- name: verify the collection against the first valid server
+  command: ansible-galaxy collection verify ansible_test.verify:1.0.0 -vvv {{ galaxy_verbosity }}
+  environment:
+    ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}'
+  register: verify
+
+- assert:
+    that:
+      - verify is success
+      - >-
+        "Found API version '{{ test_api_fallback_versions }}' with Galaxy server {{ test_api_fallback }}" in verify.stdout
+
 - name: verify the installed collection against the server
-  command: ansible-galaxy collection verify ansible_test.verify:1.0.0
+  command: ansible-galaxy collection verify ansible_test.verify:1.0.0 -s {{ test_name }} {{ galaxy_verbosity }}
   environment:
     ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}'
   register: verify
@@ -41,12 +53,12 @@
       - "'Collection ansible_test.verify contains modified content' not in verify.stdout"
 
 - name: verify the installed collection against the server, with unspecified version in CLI
-  command: ansible-galaxy collection verify ansible_test.verify
+  command: ansible-galaxy collection verify ansible_test.verify -s {{ test_name }} {{ galaxy_verbosity }}
   environment:
     ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}'
 
 - name: verify a collection that doesn't appear to be installed
-  command: ansible-galaxy collection verify ansible_test.verify:1.0.0
+  command: ansible-galaxy collection verify ansible_test.verify:1.0.0 -s {{ test_name }} {{ galaxy_verbosity }}
   register: verify
   failed_when: verify.rc == 0
 
@@ -82,7 +94,7 @@
     chdir: '{{ galaxy_dir }}'
 
 - name: verify a version of a collection that isn't installed
-  command: ansible-galaxy collection verify ansible_test.verify:2.0.0
+  command: ansible-galaxy collection verify ansible_test.verify:2.0.0 -s {{ test_name }} {{ galaxy_verbosity }}
   environment:
     ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}'
   register: verify
@@ -94,12 +106,12 @@
       - '"ansible_test.verify has the version ''1.0.0'' but is being compared to ''2.0.0''" in verify.stdout'
 
 - name: install the new version from the server
-  command: ansible-galaxy collection install ansible_test.verify:2.0.0 --force
+  command: ansible-galaxy collection install ansible_test.verify:2.0.0 --force -s {{ test_name }} {{ galaxy_verbosity }}
   environment:
     ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}'
 
 - name: verify the installed collection against the server
-  command: ansible-galaxy collection verify ansible_test.verify:2.0.0
+  command: ansible-galaxy collection verify ansible_test.verify:2.0.0 -s {{ test_name }} {{ galaxy_verbosity }}
   environment:
     ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}'
   register: verify
@@ -145,7 +157,7 @@
       - "updated_file.stat.checksum != file.stat.checksum"
 
 - name: test verifying checksumes of the modified collection
-  command: ansible-galaxy collection verify ansible_test.verify:2.0.0
+  command: ansible-galaxy collection verify ansible_test.verify:2.0.0 -s {{ test_name }} {{ galaxy_verbosity }}
   register: verify
   environment:
     ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}'
@@ -165,7 +177,7 @@
   diff: true
 
 - name: ensure a modified FILES.json is validated
-  command: ansible-galaxy collection verify ansible_test.verify:2.0.0
+  command: ansible-galaxy collection verify ansible_test.verify:2.0.0 -s {{ test_name }} {{ galaxy_verbosity }}
   register: verify
   environment:
     ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}'
@@ -189,7 +201,7 @@
     line: ' "chksum_sha256": "{{ manifest_info.stat.checksum }}",'
 
 - name: ensure the MANIFEST.json is validated against the uncorrupted file from the server
-  command: ansible-galaxy collection verify ansible_test.verify:2.0.0
+  command: ansible-galaxy collection verify ansible_test.verify:2.0.0 -s {{ test_name }} {{ galaxy_verbosity }}
   register: verify
   environment:
     ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}'
@@ -219,7 +231,7 @@
     dest: '{{ galaxy_dir }}/ansible_collections/ansible_test/verify/galaxy.yml'
 
 - name: test we only verify collections containing a MANIFEST.json with the version on the server
-  command: ansible-galaxy collection verify ansible_test.verify:2.0.0
+  command: ansible-galaxy collection verify ansible_test.verify:2.0.0 -s {{ test_name }} {{ galaxy_verbosity }}
   register: verify
   environment:
     ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}'
diff --git a/test/integration/targets/ansible-galaxy-collection/templates/ansible.cfg.j2 b/test/integration/targets/ansible-galaxy-collection/templates/ansible.cfg.j2
index 62f3dcf901..00c1fe4deb 100644
--- a/test/integration/targets/ansible-galaxy-collection/templates/ansible.cfg.j2
+++ b/test/integration/targets/ansible-galaxy-collection/templates/ansible.cfg.j2
@@ -1,7 +1,10 @@
 [galaxy]
 # Ensures subsequent unstable reruns don't use the cached information causing another failure
 cache_dir={{ remote_tmp_dir }}/galaxy_cache
-server_list=pulp_v2,pulp_v3,galaxy_ng,secondary
+server_list=offline,pulp_v2,pulp_v3,galaxy_ng,secondary
+
+[galaxy_server.offline]
+url=https://test-hub.demolab.local/api/galaxy/content/api/
 
 [galaxy_server.pulp_v2]
 url={{ pulp_server }}published/api/