ansible-galaxy - define multiple galaxy instances in ansible.cfg (#60553)

* ansible-galaxy: support multiple servers on install

* Added docs for the server configuration file

* Fix up doc string for requirements file format

* Fix bugs after testing

* Fix kwarg doc and added version

* Fix typo and doc improvement

* Fix base64 encoding and allow --server to override list

1 files changed, 72 insertions, 142 deletions

diff --git a/test/units/galaxy/test_collection.py b/test/units/galaxy/test_collection.py
index 2f7c498aff..437a5e985a 100644
--- a/test/units/galaxy/test_collection.py
+++ b/test/units/galaxy/test_collection.py
@@ -21,9 +21,10 @@ from units.compat.mock import MagicMock
 
 import ansible.module_utils.six.moves.urllib.error as urllib_error
 
+from ansible import context
 from ansible.cli.galaxy import GalaxyCLI
 from ansible.errors import AnsibleError
-from ansible.galaxy import collection
+from ansible.galaxy import api, collection, token
 from ansible.module_utils._text import to_bytes, to_native, to_text
 from ansible.utils import context_objects as co
 from ansible.utils.display import Display
@@ -78,20 +79,6 @@ def collection_artifact(monkeypatch, tmp_path_factory):
 
 
 @pytest.fixture()
-def requirements_file(request, tmp_path_factory):
-    content = request.param
-
-    test_dir = to_text(tmp_path_factory.mktemp('test-ÅÑŚÌβŁÈ Collections Requirements'))
-    requirements_file = os.path.join(test_dir, 'requirements.yml')
-
-    if content:
-        with open(requirements_file, 'wb') as req_obj:
-            req_obj.write(to_bytes(content))
-
-    yield requirements_file
-
-
-@pytest.fixture()
 def galaxy_yml(request, tmp_path_factory):
     b_test_dir = to_bytes(tmp_path_factory.mktemp('test-ÅÑŚÌβŁÈ Collections'))
     b_galaxy_yml = os.path.join(b_test_dir, b'galaxy.yml')
@@ -123,6 +110,14 @@ def tmp_tarfile(tmp_path_factory):
         yield temp_dir, tfile, filename, sha256_hash.hexdigest()
 
 
+@pytest.fixture()
+def galaxy_server():
+    context.CLIARGS._store = {'ignore_certs': False}
+    galaxy_api = api.GalaxyAPI(None, 'test_server', 'https://galaxy.ansible.com',
+                               token=token.GalaxyToken(token='key'))
+    return galaxy_api
+
+
 def test_build_collection_no_galaxy_yaml():
     fake_path = u'/fake/ÅÑŚÌβŁÈ/path'
     expected = to_native("The collection galaxy.yml path '%s/galaxy.yml' does not exist." % fake_path)
@@ -411,7 +406,7 @@ def test_publish_missing_file():
     expected = to_native("The collection path specified '%s' does not exist." % fake_path)
 
     with pytest.raises(AnsibleError, match=expected):
-        collection.publish_collection(fake_path, None, None, False, True)
+        collection.publish_collection(fake_path, None, True)
 
 
 def test_publish_not_a_tarball():
@@ -422,24 +417,23 @@ def test_publish_not_a_tarball():
         temp_file.write(b"\x00")
         temp_file.flush()
         with pytest.raises(AnsibleError, match=expected.format(to_native(temp_file.name))):
-            collection.publish_collection(temp_file.name, None, None, False, True)
+            collection.publish_collection(temp_file.name, None, True)
 
 
-def test_publish_no_wait(collection_artifact, monkeypatch):
+def test_publish_no_wait(galaxy_server, collection_artifact, monkeypatch):
     mock_display = MagicMock()
     monkeypatch.setattr(Display, 'display', mock_display)
 
     artifact_path, mock_open = collection_artifact
     fake_import_uri = 'https://galaxy.server.com/api/v2/import/1234'
-    server = 'https://galaxy.com'
 
     mock_open.return_value = StringIO(u'{"task":"%s"}' % fake_import_uri)
     expected_form, expected_content_type = collection._get_mime_data(to_bytes(artifact_path))
 
-    collection.publish_collection(artifact_path, server, 'key', False, False)
+    collection.publish_collection(artifact_path, galaxy_server, False)
 
     assert mock_open.call_count == 1
-    assert mock_open.mock_calls[0][1][0] == 'https://galaxy.com/api/v2/collections/'
+    assert mock_open.mock_calls[0][1][0] == '%s/api/v2/collections/' % galaxy_server.api_server
     assert mock_open.mock_calls[0][2]['data'] == expected_form
     assert mock_open.mock_calls[0][2]['method'] == 'POST'
     assert mock_open.mock_calls[0][2]['validate_certs'] is True
@@ -448,24 +442,26 @@ def test_publish_no_wait(collection_artifact, monkeypatch):
     assert mock_open.mock_calls[0][2]['headers']['Content-type'] == expected_content_type
 
     assert mock_display.call_count == 2
-    assert mock_display.mock_calls[0][1][0] == "Publishing collection artifact '%s' to %s" % (artifact_path, server)
+    assert mock_display.mock_calls[0][1][0] == "Publishing collection artifact '%s' to %s %s" \
+        % (artifact_path, galaxy_server.name, galaxy_server.api_server)
     assert mock_display.mock_calls[1][1][0] == \
         "Collection has been pushed to the Galaxy server, not waiting until import has completed due to --no-wait " \
         "being set. Import task results can be found at %s" % fake_import_uri
 
 
-def test_publish_dont_validate_cert(collection_artifact):
+def test_publish_dont_validate_cert(galaxy_server, collection_artifact):
+    galaxy_server.validate_certs = False
     artifact_path, mock_open = collection_artifact
 
     mock_open.return_value = StringIO(u'{"task":"https://galaxy.server.com/api/v2/import/1234"}')
 
-    collection.publish_collection(artifact_path, 'https://galaxy.server.com', 'key', True, False)
+    collection.publish_collection(artifact_path, galaxy_server, False)
 
     assert mock_open.call_count == 1
     assert mock_open.mock_calls[0][2]['validate_certs'] is False
 
 
-def test_publish_failure(collection_artifact):
+def test_publish_failure(galaxy_server, collection_artifact):
     artifact_path, mock_open = collection_artifact
 
     mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com', 500, 'msg', {}, StringIO())
@@ -473,10 +469,10 @@ def test_publish_failure(collection_artifact):
     expected = 'Error when publishing collection (HTTP Code: 500, Message: Unknown error returned by Galaxy ' \
                'server. Code: Unknown)'
     with pytest.raises(AnsibleError, match=re.escape(expected)):
-        collection.publish_collection(artifact_path, 'https://galaxy.server.com', 'key', False, True)
+        collection.publish_collection(artifact_path, galaxy_server, True)
 
 
-def test_publish_failure_with_json_info(collection_artifact):
+def test_publish_failure_with_json_info(galaxy_server, collection_artifact):
     artifact_path, mock_open = collection_artifact
 
     return_content = StringIO(u'{"message":"Galaxy error message","code":"GWE002"}')
@@ -484,10 +480,10 @@ def test_publish_failure_with_json_info(collection_artifact):
 
     expected = 'Error when publishing collection (HTTP Code: 503, Message: Galaxy error message Code: GWE002)'
     with pytest.raises(AnsibleError, match=re.escape(expected)):
-        collection.publish_collection(artifact_path, 'https://galaxy.server.com', 'key', False, True)
+        collection.publish_collection(artifact_path, galaxy_server, True)
 
 
-def test_publish_with_wait(collection_artifact, monkeypatch):
+def test_publish_with_wait(galaxy_server, collection_artifact, monkeypatch):
     mock_display = MagicMock()
     monkeypatch.setattr(Display, 'display', mock_display)
 
@@ -495,7 +491,6 @@ def test_publish_with_wait(collection_artifact, monkeypatch):
     monkeypatch.setattr(Display, 'vvv', mock_vvv)
 
     fake_import_uri = 'https://galaxy-server/api/v2/import/1234'
-    server = 'https://galaxy.server.com'
 
     artifact_path, mock_open = collection_artifact
 
@@ -504,7 +499,7 @@ def test_publish_with_wait(collection_artifact, monkeypatch):
         StringIO(u'{"finished_at":"some_time","state":"success"}')
     )
 
-    collection.publish_collection(artifact_path, server, 'key', False, True)
+    collection.publish_collection(artifact_path, galaxy_server, True)
 
     assert mock_open.call_count == 2
     assert mock_open.mock_calls[1][1][0] == fake_import_uri
@@ -513,12 +508,14 @@ def test_publish_with_wait(collection_artifact, monkeypatch):
     assert mock_open.mock_calls[1][2]['method'] == 'GET'
 
     assert mock_display.call_count == 2
-    assert mock_display.mock_calls[0][1][0] == "Publishing collection artifact '%s' to %s" % (artifact_path, server)
+    assert mock_display.mock_calls[0][1][0] == "Publishing collection artifact '%s' to %s %s" \
+        % (artifact_path, galaxy_server.name, galaxy_server.api_server)
     assert mock_display.mock_calls[1][1][0] == 'Collection has been successfully published to the Galaxy server'
 
-    assert mock_vvv.call_count == 2
-    assert mock_vvv.mock_calls[0][1][0] == 'Collection has been pushed to the Galaxy server %s' % server
-    assert mock_vvv.mock_calls[1][1][0] == 'Waiting until galaxy import task %s has completed' % fake_import_uri
+    assert mock_vvv.call_count == 3
+    assert mock_vvv.mock_calls[1][1][0] == 'Collection has been pushed to the Galaxy server %s %s' \
+        % (galaxy_server.name, galaxy_server.api_server)
+    assert mock_vvv.mock_calls[2][1][0] == 'Waiting until galaxy import task %s has completed' % fake_import_uri
 
 
 def test_publish_with_wait_timeout(collection_artifact, monkeypatch):
@@ -564,7 +561,9 @@ def test_publish_with_wait_timeout(collection_artifact, monkeypatch):
         'Galaxy import process has a status of waiting, wait 2 seconds before trying again'
 
 
-def test_publish_with_wait_timeout(collection_artifact, monkeypatch):
+def test_publish_with_wait_timeout(galaxy_server, collection_artifact, monkeypatch):
+    galaxy_server.validate_certs = False
+
     monkeypatch.setattr(time, 'sleep', MagicMock())
 
     mock_display = MagicMock()
@@ -574,7 +573,6 @@ def test_publish_with_wait_timeout(collection_artifact, monkeypatch):
     monkeypatch.setattr(Display, 'vvv', mock_vvv)
 
     fake_import_uri = 'https://galaxy-server/api/v2/import/1234'
-    server = 'https://galaxy.server.com'
 
     artifact_path, mock_open = collection_artifact
 
@@ -592,7 +590,7 @@ def test_publish_with_wait_timeout(collection_artifact, monkeypatch):
     expected = "Timeout while waiting for the Galaxy import process to finish, check progress at '%s'" \
         % fake_import_uri
     with pytest.raises(AnsibleError, match=expected):
-        collection.publish_collection(artifact_path, server, 'key', True, True)
+        collection.publish_collection(artifact_path, galaxy_server, True)
 
     assert mock_open.call_count == 8
     for i in range(7):
@@ -603,21 +601,23 @@ def test_publish_with_wait_timeout(collection_artifact, monkeypatch):
         assert mock_call[2]['method'] == 'GET'
 
     assert mock_display.call_count == 1
-    assert mock_display.mock_calls[0][1][0] == "Publishing collection artifact '%s' to %s" % (artifact_path, server)
+    assert mock_display.mock_calls[0][1][0] == "Publishing collection artifact '%s' to %s %s" \
+        % (artifact_path, galaxy_server.name, galaxy_server.api_server)
 
     expected_wait_msg = 'Galaxy import process has a status of waiting, wait {0} seconds before trying again'
-    assert mock_vvv.call_count == 8
-    assert mock_vvv.mock_calls[0][1][0] == 'Collection has been pushed to the Galaxy server %s' % server
-    assert mock_vvv.mock_calls[1][1][0] == 'Waiting until galaxy import task %s has completed' % fake_import_uri
-    assert mock_vvv.mock_calls[2][1][0] == expected_wait_msg.format(2)
-    assert mock_vvv.mock_calls[3][1][0] == expected_wait_msg.format(3)
-    assert mock_vvv.mock_calls[4][1][0] == expected_wait_msg.format(4)
-    assert mock_vvv.mock_calls[5][1][0] == expected_wait_msg.format(6)
-    assert mock_vvv.mock_calls[6][1][0] == expected_wait_msg.format(10)
-    assert mock_vvv.mock_calls[7][1][0] == expected_wait_msg.format(15)
-
-
-def test_publish_with_wait_and_failure(collection_artifact, monkeypatch):
+    assert mock_vvv.call_count == 9
+    assert mock_vvv.mock_calls[1][1][0] == 'Collection has been pushed to the Galaxy server %s %s' \
+        % (galaxy_server.name, galaxy_server.api_server)
+    assert mock_vvv.mock_calls[2][1][0] == 'Waiting until galaxy import task %s has completed' % fake_import_uri
+    assert mock_vvv.mock_calls[3][1][0] == expected_wait_msg.format(2)
+    assert mock_vvv.mock_calls[4][1][0] == expected_wait_msg.format(3)
+    assert mock_vvv.mock_calls[5][1][0] == expected_wait_msg.format(4)
+    assert mock_vvv.mock_calls[6][1][0] == expected_wait_msg.format(6)
+    assert mock_vvv.mock_calls[7][1][0] == expected_wait_msg.format(10)
+    assert mock_vvv.mock_calls[8][1][0] == expected_wait_msg.format(15)
+
+
+def test_publish_with_wait_and_failure(galaxy_server, collection_artifact, monkeypatch):
     mock_display = MagicMock()
     monkeypatch.setattr(Display, 'display', mock_display)
 
@@ -631,7 +631,6 @@ def test_publish_with_wait_and_failure(collection_artifact, monkeypatch):
     monkeypatch.setattr(Display, 'error', mock_err)
 
     fake_import_uri = 'https://galaxy-server/api/v2/import/1234'
-    server = 'https://galaxy.server.com'
 
     artifact_path, mock_open = collection_artifact
 
@@ -666,21 +665,23 @@ def test_publish_with_wait_and_failure(collection_artifact, monkeypatch):
 
     expected = 'Galaxy import process failed: Because I said so! (Code: GW001)'
     with pytest.raises(AnsibleError, match=re.escape(expected)):
-        collection.publish_collection(artifact_path, server, 'key', True, True)
+        collection.publish_collection(artifact_path, galaxy_server, True)
 
     assert mock_open.call_count == 2
     assert mock_open.mock_calls[1][1][0] == fake_import_uri
     assert mock_open.mock_calls[1][2]['headers']['Authorization'] == 'Token key'
-    assert mock_open.mock_calls[1][2]['validate_certs'] is False
+    assert mock_open.mock_calls[1][2]['validate_certs'] is True
     assert mock_open.mock_calls[1][2]['method'] == 'GET'
 
     assert mock_display.call_count == 1
-    assert mock_display.mock_calls[0][1][0] == "Publishing collection artifact '%s' to %s" % (artifact_path, server)
+    assert mock_display.mock_calls[0][1][0] == "Publishing collection artifact '%s' to %s %s" \
+        % (artifact_path, galaxy_server.name, galaxy_server.api_server)
 
-    assert mock_vvv.call_count == 3
-    assert mock_vvv.mock_calls[0][1][0] == 'Collection has been pushed to the Galaxy server %s' % server
-    assert mock_vvv.mock_calls[1][1][0] == 'Waiting until galaxy import task %s has completed' % fake_import_uri
-    assert mock_vvv.mock_calls[2][1][0] == 'Galaxy import message: info - Some info'
+    assert mock_vvv.call_count == 4
+    assert mock_vvv.mock_calls[1][1][0] == 'Collection has been pushed to the Galaxy server %s %s' \
+        % (galaxy_server.name, galaxy_server.api_server)
+    assert mock_vvv.mock_calls[2][1][0] == 'Waiting until galaxy import task %s has completed' % fake_import_uri
+    assert mock_vvv.mock_calls[3][1][0] == 'Galaxy import message: info - Some info'
 
     assert mock_warn.call_count == 1
     assert mock_warn.mock_calls[0][1][0] == 'Galaxy import warning message: Some warning'
@@ -689,7 +690,7 @@ def test_publish_with_wait_and_failure(collection_artifact, monkeypatch):
     assert mock_err.mock_calls[0][1][0] == 'Galaxy import error message: Some error'
 
 
-def test_publish_with_wait_and_failure_and_no_error(collection_artifact, monkeypatch):
+def test_publish_with_wait_and_failure_and_no_error(galaxy_server, collection_artifact, monkeypatch):
     mock_display = MagicMock()
     monkeypatch.setattr(Display, 'display', mock_display)
 
@@ -703,7 +704,6 @@ def test_publish_with_wait_and_failure_and_no_error(collection_artifact, monkeyp
     monkeypatch.setattr(Display, 'error', mock_err)
 
     fake_import_uri = 'https://galaxy-server/api/v2/import/1234'
-    server = 'https://galaxy.server.com'
 
     artifact_path, mock_open = collection_artifact
 
@@ -734,21 +734,23 @@ def test_publish_with_wait_and_failure_and_no_error(collection_artifact, monkeyp
 
     expected = 'Galaxy import process failed: Unknown error, see %s for more details (Code: UNKNOWN)' % fake_import_uri
     with pytest.raises(AnsibleError, match=re.escape(expected)):
-        collection.publish_collection(artifact_path, server, 'key', True, True)
+        collection.publish_collection(artifact_path, galaxy_server, True)
 
     assert mock_open.call_count == 2
     assert mock_open.mock_calls[1][1][0] == fake_import_uri
     assert mock_open.mock_calls[1][2]['headers']['Authorization'] == 'Token key'
-    assert mock_open.mock_calls[1][2]['validate_certs'] is False
+    assert mock_open.mock_calls[1][2]['validate_certs'] is True
     assert mock_open.mock_calls[1][2]['method'] == 'GET'
 
     assert mock_display.call_count == 1
-    assert mock_display.mock_calls[0][1][0] == "Publishing collection artifact '%s' to %s" % (artifact_path, server)
+    assert mock_display.mock_calls[0][1][0] == "Publishing collection artifact '%s' to %s %s" \
+        % (artifact_path, galaxy_server.name, galaxy_server.api_server)
 
-    assert mock_vvv.call_count == 3
-    assert mock_vvv.mock_calls[0][1][0] == 'Collection has been pushed to the Galaxy server %s' % server
-    assert mock_vvv.mock_calls[1][1][0] == 'Waiting until galaxy import task %s has completed' % fake_import_uri
-    assert mock_vvv.mock_calls[2][1][0] == 'Galaxy import message: info - Some info'
+    assert mock_vvv.call_count == 4
+    assert mock_vvv.mock_calls[1][1][0] == 'Collection has been pushed to the Galaxy server %s %s' \
+        % (galaxy_server.name, galaxy_server.api_server)
+    assert mock_vvv.mock_calls[2][1][0] == 'Waiting until galaxy import task %s has completed' % fake_import_uri
+    assert mock_vvv.mock_calls[3][1][0] == 'Galaxy import message: info - Some info'
 
     assert mock_warn.call_count == 1
     assert mock_warn.mock_calls[0][1][0] == 'Galaxy import warning message: Some warning'
@@ -757,78 +759,6 @@ def test_publish_with_wait_and_failure_and_no_error(collection_artifact, monkeyp
     assert mock_err.mock_calls[0][1][0] == 'Galaxy import error message: Some error'
 
 
-@pytest.mark.parametrize('requirements_file', [None], indirect=True)
-def test_parse_requirements_file_that_doesnt_exist(requirements_file):
-    expected = "The requirements file '%s' does not exist." % to_native(requirements_file)
-    with pytest.raises(AnsibleError, match=expected):
-        collection.parse_collections_requirements_file(requirements_file)
-
-
-@pytest.mark.parametrize('requirements_file', ['not a valid yml file: hi: world'], indirect=True)
-def test_parse_requirements_file_that_isnt_yaml(requirements_file):
-    expected = "Failed to parse the collection requirements yml at '%s' with the following error" \
-               % to_native(requirements_file)
-    with pytest.raises(AnsibleError, match=expected):
-        collection.parse_collections_requirements_file(requirements_file)
-
-
-@pytest.mark.parametrize('requirements_file', [('''
-# Older role based requirements.yml
-- galaxy.role
-- anotherrole
-'''), ('''
-# Doesn't have collections key
-roles:
-- galaxy.role
-- anotherole
-''')], indirect=True)
-def test_parse_requirements_in_invalid_format(requirements_file):
-    expected = "Expecting collections requirements file to be a dict with the key collections that contains a list " \
-               "of collections to install."
-    with pytest.raises(AnsibleError, match=expected):
-        collection.parse_collections_requirements_file(requirements_file)
-
-
-@pytest.mark.parametrize('requirements_file', ['''
-collections:
-- version: 1.0.0
-'''], indirect=True)
-def test_parse_requirements_without_mandatory_name_key(requirements_file):
-    expected = "Collections requirement entry should contain the key name."
-    with pytest.raises(AnsibleError, match=expected):
-        collection.parse_collections_requirements_file(requirements_file)
-
-
-@pytest.mark.parametrize('requirements_file', [('''
-collections:
-- namespace.collection1
-- namespace.collection2
-'''), ('''
-collections:
-- name: namespace.collection1
-- name: namespace.collection2
-''')], indirect=True)
-def test_parse_requirements(requirements_file):
-    expected = [('namespace.collection1', '*', None), ('namespace.collection2', '*', None)]
-    actual = collection.parse_collections_requirements_file(requirements_file)
-
-    assert actual == expected
-
-
-@pytest.mark.parametrize('requirements_file', ['''
-collections:
-- name: namespace.collection1
-  version: ">=1.0.0,<=2.0.0"
-  source: https://galaxy-dev.ansible.com
-- namespace.collection2'''], indirect=True)
-def test_parse_requirements_with_extra_info(requirements_file):
-    expected = [('namespace.collection1', '>=1.0.0,<=2.0.0', 'https://galaxy-dev.ansible.com'),
-                ('namespace.collection2', '*', None)]
-    actual = collection.parse_collections_requirements_file(requirements_file)
-
-    assert actual == expected
-
-
 def test_find_existing_collections(tmp_path_factory, monkeypatch):
     test_dir = to_text(tmp_path_factory.mktemp('test-ÅÑŚÌβŁÈ Collections'))
     collection1 = os.path.join(test_dir, 'namespace1', 'collection1')
@@ -869,7 +799,7 @@ def test_find_existing_collections(tmp_path_factory, monkeypatch):
             assert actual_collection.namespace == 'namespace1'
             assert actual_collection.name == 'collection1'
             assert actual_collection.b_path == to_bytes(collection1)
-            assert actual_collection.source is None
+            assert actual_collection.api is None
             assert actual_collection.versions == set(['1.2.3'])
             assert actual_collection.latest_version == '1.2.3'
             assert actual_collection.dependencies == {}
@@ -877,7 +807,7 @@ def test_find_existing_collections(tmp_path_factory, monkeypatch):
             assert actual_collection.namespace == 'namespace2'
             assert actual_collection.name == 'collection2'
             assert actual_collection.b_path == to_bytes(collection2)
-            assert actual_collection.source is None
+            assert actual_collection.api is None
             assert actual_collection.versions == set(['*'])
             assert actual_collection.latest_version == '*'
             assert actual_collection.dependencies == {}