diff options
author | Jordan Borean <jborean93@gmail.com> | 2020-03-03 08:26:27 +1000 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-03-02 14:26:27 -0800 |
commit | 0f7d62f6a531a8af135bca8aac27838c2847eb9e (patch) | |
tree | 9908f0c1459bc9c7184ae38faec01f12697f7702 | |
parent | b38603c45ed3a53574ec2080fb3a24db38ab5bc6 (diff) | |
download | ansible-0f7d62f6a531a8af135bca8aac27838c2847eb9e.tar.gz |
ansible-galaxy - optimise some paths and use fake galaxy int tests (#67685) - 2.9 (#67874)
* ansible-galaxy - optimise some paths and use fake galaxy int tests (#67685)
* ansible-galaxy - optimise some paths and use fake galaxy int tests
* Added init, built, and publish tests
* Test against both mocked Galaxy and AH server
* Finish off writing the install tests
* Fix up broken tests
* Rename test target and add migrated tests
* Use cloud provider for Galaxy implementation
* Added blank static config
* Use correct alias group
* Set release version and fix copy typo
* Remove reset step as it is no longer needed
* Use sane env var names for test container name
(cherry picked from commit 26129fcb8056220a9f01ef14a12784a7bb4c4327)
* Use --api-key and not --token
* Set fallaxy tests as a smoketest
(cherry picked from commit b241c021b75284e79c36c3ce1efa6ba5279d6d4c)
15 files changed, 866 insertions, 136 deletions
diff --git a/changelogs/fragments/ansible-galaxy-collections.yaml b/changelogs/fragments/ansible-galaxy-collections.yaml new file mode 100644 index 0000000000..a52b40c336 --- /dev/null +++ b/changelogs/fragments/ansible-galaxy-collections.yaml @@ -0,0 +1,5 @@ +bugfixes: +- ansible-galaxy - Remove uneeded verbose messages when accessing local token file +- ansible-galaxy - Display proper error when invalid token is used for Galaxy servers +- ansible-galaxy - Send SHA256 hashes when publishing a collection +- ansible-galaxy - Fix up pagination searcher for collection versions on Automation Hub diff --git a/lib/ansible/galaxy/api.py b/lib/ansible/galaxy/api.py index 8874234945..a224a305b7 100644 --- a/lib/ansible/galaxy/api.py +++ b/lib/ansible/galaxy/api.py @@ -5,6 +5,7 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type +import hashlib import json import os import tarfile @@ -53,19 +54,27 @@ def g_connect(versions): try: data = self._call_galaxy(n_url, method='GET', error_context_msg=error_context_msg) - except (AnsibleError, GalaxyError, ValueError, KeyError): + except (AnsibleError, GalaxyError, ValueError, KeyError) as err: # Either the URL doesnt exist, or other error. Or the URL exists, but isn't a galaxy API # root (not JSON, no 'available_versions') so try appending '/api/' - n_url = _urljoin(n_url, '/api/') - - # let exceptions here bubble up - data = self._call_galaxy(n_url, method='GET', error_context_msg=error_context_msg) - if 'available_versions' not in data: - raise AnsibleError("Tried to find galaxy API root at %s but no 'available_versions' are available on %s" - % (n_url, self.api_server)) + if n_url.endswith('/api') or n_url.endswith('/api/'): + raise - # Update api_server to point to the "real" API root, which in this case - # was the configured url + '/api/' appended. + # Let exceptions here bubble up but raise the original if this returns a 404 (/api/ wasn't found). + n_url = _urljoin(n_url, '/api/') + try: + data = self._call_galaxy(n_url, method='GET', error_context_msg=error_context_msg) + except GalaxyError as new_err: + if new_err.http_code == 404: + raise err + raise + + if 'available_versions' not in data: + raise AnsibleError("Tried to find galaxy API root at %s but no 'available_versions' are available " + "on %s" % (n_url, self.api_server)) + + # Update api_server to point to the "real" API root, which in this case could have been the configured + # url + '/api/' appended. self.api_server = n_url # Default to only supporting v1, if only v1 is returned we also assume that v2 is available even though @@ -185,7 +194,7 @@ class GalaxyAPI: try: display.vvvv("Calling Galaxy at %s" % url) resp = open_url(to_native(url), data=args, validate_certs=self.validate_certs, headers=headers, - method=method, timeout=20, http_agent=user_agent()) + method=method, timeout=20, http_agent=user_agent(), follow_redirects='safe') except HTTPError as e: raise GalaxyError(e, error_context_msg) except Exception as e: @@ -426,7 +435,7 @@ class GalaxyAPI: form = [ part_boundary, b"Content-Disposition: form-data; name=\"sha256\"", - to_bytes(secure_hash_s(data), errors='surrogate_or_strict'), + to_bytes(secure_hash_s(data, hash_func=hashlib.sha256), errors='surrogate_or_strict'), part_boundary, b"Content-Disposition: file; name=\"file\"; filename=\"%s\"" % b_file_name, b"Content-Type: application/octet-stream", @@ -460,7 +469,6 @@ class GalaxyAPI: value for GalaxyAPI.publish_collection. :param timeout: The timeout in seconds, 0 is no timeout. """ - # TODO: actually verify that v3 returns the same structure as v2, right now this is just an assumption. state = 'waiting' data = None @@ -469,10 +477,8 @@ class GalaxyAPI: full_url = _urljoin(self.api_server, self.available_api_versions['v3'], 'imports/collections', task_id, '/') else: - # TODO: Should we have a trailing slash here? I'm working with what the unittests ask - # for but a trailing slash may be more correct full_url = _urljoin(self.api_server, self.available_api_versions['v2'], - 'collection-imports', task_id) + 'collection-imports', task_id, '/') display.display("Waiting until Galaxy import task %s has completed" % full_url) start = time.time() @@ -543,10 +549,12 @@ class GalaxyAPI: :param name: The collection name. :return: A list of versions that are available. """ + relative_link = False if 'v3' in self.available_api_versions: api_path = self.available_api_versions['v3'] results_key = 'data' pagination_path = ['links', 'next'] + relative_link = True # AH pagination results are relative an not an absolute URI. else: api_path = self.available_api_versions['v2'] results_key = 'results' @@ -568,6 +576,10 @@ class GalaxyAPI: if not next_link: break + elif relative_link: + # TODO: This assumes the pagination result is relative to the root server. Will need to be verified + # with someone who knows the AH API. + next_link = n_url.replace(urlparse(n_url).path, next_link) data = self._call_galaxy(to_native(next_link, errors='surrogate_or_strict'), error_context_msg=error_context_msg) diff --git a/lib/ansible/galaxy/token.py b/lib/ansible/galaxy/token.py index 31f2d88e74..7231c8f9a4 100644 --- a/lib/ansible/galaxy/token.py +++ b/lib/ansible/galaxy/token.py @@ -108,7 +108,7 @@ class GalaxyToken(object): @property def config(self): - if not self._config: + if self._config is None: self._config = self._read() # Prioritise the token passed into the constructor diff --git a/test/integration/targets/ansible-galaxy-collection/aliases b/test/integration/targets/ansible-galaxy-collection/aliases new file mode 100644 index 0000000000..8af920aef1 --- /dev/null +++ b/test/integration/targets/ansible-galaxy-collection/aliases @@ -0,0 +1,3 @@ +shippable/cloud/group1 +shippable/cloud/smoketest +cloud/fallaxy diff --git a/test/integration/targets/ansible-galaxy-collection/library/setup_collections.py b/test/integration/targets/ansible-galaxy-collection/library/setup_collections.py new file mode 100644 index 0000000000..9129ca768f --- /dev/null +++ b/test/integration/targets/ansible-galaxy-collection/library/setup_collections.py @@ -0,0 +1,150 @@ +#!/usr/bin/python + +# Copyright: (c) 2020, Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +ANSIBLE_METADATA = { + 'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community' +} + +DOCUMENTATION = ''' +--- +module: setup_collections +short_description: Set up test collections based on the input +description: +- Builds and publishes a whole bunch of collections used for testing in bulk. +options: + server: + description: + - The Galaxy server to upload the collections to. + required: yes + type: str + token: + description: + - The token used to authenticate with the Galaxy server. + required: yes + type: str + collections: + description: + - A list of collection details to use for the build. + required: yes + type: list + elements: dict + options: + namespace: + description: + - The namespace of the collection. + required: yes + type: str + name: + description: + - The name of the collection. + required: yes + type: str + version: + description: + - The version of the collection. + type: str + default: '1.0.0' + dependencies: + description: + - The dependencies of the collection. + type: dict + default: '{}' +author: +- Jordan Borean (@jborean93) +''' + +EXAMPLES = ''' +- name: Build test collections + setup_collections: + path: ~/ansible/collections/ansible_collections + collections: + - namespace: namespace1 + name: name1 + version: 0.0.1 + - namespace: namespace1 + name: name1 + version: 0.0.2 +''' + +RETURN = ''' +# +''' + +import os +import yaml + +from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils._text import to_bytes + + +def run_module(): + module_args = dict( + server=dict(type='str', required=True), + token=dict(type='str', required=True), + collections=dict( + type='list', + elements='dict', + required=True, + options=dict( + namespace=dict(type='str', required=True), + name=dict(type='str', required=True), + version=dict(type='str', default='1.0.0'), + dependencies=dict(type='dict', default={}), + ), + ), + ) + + module = AnsibleModule( + argument_spec=module_args, + supports_check_mode=False + ) + + result = dict(changed=True) + + for idx, collection in enumerate(module.params['collections']): + collection_dir = os.path.join(module.tmpdir, "%s-%s-%s" % (collection['namespace'], collection['name'], + collection['version'])) + b_collection_dir = to_bytes(collection_dir, errors='surrogate_or_strict') + os.mkdir(b_collection_dir) + + with open(os.path.join(b_collection_dir, b'README.md'), mode='wb') as fd: + fd.write(b"Collection readme") + + galaxy_meta = { + 'namespace': collection['namespace'], + 'name': collection['name'], + 'version': collection['version'], + 'readme': 'README.md', + 'authors': ['Collection author <name@email.com'], + 'dependencies': collection['dependencies'], + } + with open(os.path.join(b_collection_dir, b'galaxy.yml'), mode='wb') as fd: + fd.write(to_bytes(yaml.safe_dump(galaxy_meta), errors='surrogate_or_strict')) + + release_filename = '%s-%s-%s.tar.gz' % (collection['namespace'], collection['name'], collection['version']) + collection_path = os.path.join(collection_dir, release_filename) + module.run_command(['ansible-galaxy', 'collection', 'build'], cwd=collection_dir) + + # To save on time, skip the import wait until the last collection is being uploaded. + publish_args = ['ansible-galaxy', 'collection', 'publish', collection_path, '--server', + module.params['server'], '--api-key', module.params['token']] + if idx != (len(module.params['collections']) - 1): + publish_args.append('--no-wait') + module.run_command(publish_args) + + module.exit_json(**result) + + +def main(): + run_module() + + +if __name__ == '__main__': + main() diff --git a/test/integration/targets/ansible-galaxy-collection/meta/main.yml b/test/integration/targets/ansible-galaxy-collection/meta/main.yml new file mode 100644 index 0000000000..e3dd5fb100 --- /dev/null +++ b/test/integration/targets/ansible-galaxy-collection/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: +- setup_remote_tmp_dir diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/build.yml b/test/integration/targets/ansible-galaxy-collection/tasks/build.yml new file mode 100644 index 0000000000..bd567b6d99 --- /dev/null +++ b/test/integration/targets/ansible-galaxy-collection/tasks/build.yml @@ -0,0 +1,53 @@ +--- +- name: build basic collection based on current directory + command: ansible-galaxy collection build + args: + chdir: '{{ galaxy_dir }}/scratch/ansible_test/my_collection' + register: build_current_dir + +- name: get result of build basic collection on current directory + stat: + path: '{{ galaxy_dir }}/scratch/ansible_test/my_collection/ansible_test-my_collection-1.0.0.tar.gz' + register: build_current_dir_actual + +- name: assert build basic collection based on current directory + assert: + that: + - '"Created collection for ansible_test.my_collection" in build_current_dir.stdout' + - build_current_dir_actual.stat.exists + +- name: build basic collection based on relative dir + command: ansible-galaxy collection build scratch/ansible_test/my_collection + args: + chdir: '{{ galaxy_dir }}' + register: build_relative_dir + +- name: get result of build basic collection based on relative dir + stat: + path: '{{ galaxy_dir }}/ansible_test-my_collection-1.0.0.tar.gz' + register: build_relative_dir_actual + +- name: assert build basic collection based on relative dir + assert: + that: + - '"Created collection for ansible_test.my_collection" in build_relative_dir.stdout' + - build_relative_dir_actual.stat.exists + +- name: fail to build existing collection without force + command: ansible-galaxy collection build scratch/ansible_test/my_collection + args: + chdir: '{{ galaxy_dir }}' + ignore_errors: yes + register: build_existing_no_force + +- name: build existing collection with force + command: ansible-galaxy collection build scratch/ansible_test/my_collection --force + args: + chdir: '{{ galaxy_dir }}' + register: build_existing_force + +- name: assert build existing collection + assert: + that: + - '"use --force to re-create the collection artifact" in build_existing_no_force.stderr' + - '"Created collection for ansible_test.my_collection" in build_existing_force.stdout' diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/init.yml b/test/integration/targets/ansible-galaxy-collection/tasks/init.yml new file mode 100644 index 0000000000..7a110871b5 --- /dev/null +++ b/test/integration/targets/ansible-galaxy-collection/tasks/init.yml @@ -0,0 +1,44 @@ +--- +- name: create default skeleton + command: ansible-galaxy collection init ansible_test.my_collection + args: + chdir: '{{ galaxy_dir }}/scratch' + register: init_relative + +- name: get result of create default skeleton + find: + path: '{{ galaxy_dir }}/scratch/ansible_test/my_collection' + recurse: yes + file_type: directory + register: init_relative_actual + +- debug: + var: init_relative_actual.files | map(attribute='path') | list + +- name: assert create default skeleton + assert: + that: + - '"Collection ansible_test.my_collection was created successfully" in init_relative.stdout' + - init_relative_actual.files | length == 3 + - (init_relative_actual.files | map(attribute='path') | list)[0] | basename in ['docs', 'plugins', 'roles'] + - (init_relative_actual.files | map(attribute='path') | list)[1] | basename in ['docs', 'plugins', 'roles'] + - (init_relative_actual.files | map(attribute='path') | list)[2] | basename in ['docs', 'plugins', 'roles'] + +- name: create collection with custom init path + command: ansible-galaxy collection init ansible_test2.my_collection --init-path "{{ galaxy_dir }}/scratch/custom-init-dir" + register: init_custom_path + +- name: get result of create default skeleton + find: + path: '{{ galaxy_dir }}/scratch/custom-init-dir/ansible_test2/my_collection' + file_type: directory + register: init_custom_path_actual + +- name: assert create collection with custom init path + assert: + that: + - '"Collection ansible_test2.my_collection was created successfully" in init_custom_path.stdout' + - init_custom_path_actual.files | length == 3 + - (init_custom_path_actual.files | map(attribute='path') | list)[0] | basename in ['docs', 'plugins', 'roles'] + - (init_custom_path_actual.files | map(attribute='path') | list)[1] | basename in ['docs', 'plugins', 'roles'] + - (init_custom_path_actual.files | map(attribute='path') | list)[2] | basename in ['docs', 'plugins', 'roles'] diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/install.yml b/test/integration/targets/ansible-galaxy-collection/tasks/install.yml new file mode 100644 index 0000000000..649fa8faa5 --- /dev/null +++ b/test/integration/targets/ansible-galaxy-collection/tasks/install.yml @@ -0,0 +1,181 @@ +--- +- name: create test collection install directory - {{ test_name }} + file: + path: '{{ galaxy_dir }}/ansible_collections' + state: directory + +- name: install simple collection with implicit path - {{ test_name }} + command: ansible-galaxy collection install namespace1.name1 -s '{{ test_server }}' + environment: + ANSIBLE_COLLECTIONS_PATHS: '{{ galaxy_dir }}/ansible_collections' + register: install_normal + +- name: get installed files of install simple collection with implicit path - {{ test_name }} + find: + path: '{{ galaxy_dir }}/ansible_collections/namespace1/name1' + file_type: file + register: install_normal_files + +- name: get the manifest of install simple collection with implicit path - {{ test_name }} + slurp: + path: '{{ galaxy_dir }}/ansible_collections/namespace1/name1/MANIFEST.json' + register: install_normal_manifest + +- name: assert install simple collection with implicit path - {{ test_name }} + assert: + that: + - '"Installing ''namespace1.name1:1.0.9'' to" in install_normal.stdout' + - install_normal_files.files | length == 3 + - install_normal_files.files[0].path | basename in ['MANIFEST.json', 'FILES.json', 'README.md'] + - install_normal_files.files[1].path | basename in ['MANIFEST.json', 'FILES.json', 'README.md'] + - install_normal_files.files[2].path | basename in ['MANIFEST.json', 'FILES.json', 'README.md'] + - (install_normal_manifest.content | b64decode | from_json).collection_info.version == '1.0.9' + +- name: install existing without --force - {{ test_name }} + command: ansible-galaxy collection install namespace1.name1 -s '{{ test_server }}' + environment: + ANSIBLE_COLLECTIONS_PATHS: '{{ galaxy_dir }}/ansible_collections' + register: install_existing_no_force + +- name: assert install existing without --force - {{ test_name }} + assert: + that: + - '"Skipping ''namespace1.name1'' as it is already installed" in install_existing_no_force.stdout' + +- name: install existing with --force - {{ test_name }} + command: ansible-galaxy collection install namespace1.name1 -s '{{ test_server }}' --force + environment: + ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections' + register: install_existing_force + +- name: assert install existing with --force - {{ test_name }} + assert: + that: + - '"Installing ''namespace1.name1:1.0.9'' to" in install_existing_force.stdout' + +- name: remove test installed collection - {{ test_name }} + file: + path: '{{ galaxy_dir }}/ansible_collections/namespace1' + state: absent + +- name: install pre-release as explicit version to custom dir - {{ test_name }} + command: ansible-galaxy collection install 'namespace1.name1:1.1.0-beta.1' -s '{{ test_server }}' -p '{{ galaxy_dir }}/ansible_collections' + register: install_prerelease + +- name: get result of install pre-release as explicit version to custom dir - {{ test_name }} + slurp: + path: '{{ galaxy_dir }}/ansible_collections/namespace1/name1/MANIFEST.json' + register: install_prerelease_actual + +- name: assert install pre-release as explicit version to custom dir - {{ test_name }} + assert: + that: + - '"Installing ''namespace1.name1:1.1.0-beta.1'' to" in install_prerelease.stdout' + - (install_prerelease_actual.content | b64decode | from_json).collection_info.version == '1.1.0-beta.1' + +- name: install multiple collections with dependencies - {{ test_name }} + command: ansible-galaxy collection install parent_dep.parent_collection namespace2.name -s {{ test_name }} + args: + chdir: '{{ galaxy_dir }}/ansible_collections' + environment: + ANSIBLE_COLLECTIONS_PATHS: '{{ galaxy_dir }}/ansible_collections' + ANSIBLE_CONFIG: '{{ galaxy_dir }}/ansible.cfg' + register: install_multiple_with_dep + +- name: get result of install multiple collections with dependencies - {{ test_name }} + slurp: + path: '{{ galaxy_dir }}/ansible_collections/{{ collection.namespace }}/{{ collection.name }}/MANIFEST.json' + register: install_multiple_with_dep_actual + loop_control: + loop_var: collection + loop: + - namespace: namespace2 + name: name + - namespace: parent_dep + name: parent_collection + - namespace: child_dep + name: child_collection + - namespace: child_dep + name: child_dep2 + +- name: assert install multiple collections with dependencies - {{ test_name }} + assert: + that: + - (install_multiple_with_dep_actual.results[0].content | b64decode | from_json).collection_info.version == '1.0.0' + - (install_multiple_with_dep_actual.results[1].content | b64decode | from_json).collection_info.version == '1.0.0' + - (install_multiple_with_dep_actual.results[2].content | b64decode | from_json).collection_info.version == '0.9.9' + - (install_multiple_with_dep_actual.results[3].content | b64decode | from_json).collection_info.version == '1.2.2' + +- name: expect failure with dep resolution failure + command: ansible-galaxy collection install fail_namespace.fail_collection -s {{ test_server }} + register: fail_dep_mismatch + failed_when: '"Cannot meet dependency requirement ''fail_dep2.name:<0.0.5'' for collection fail_namespace.fail_collection" not in fail_dep_mismatch.stderr' + +- name: download a collection for an offline install - {{ test_name }} + get_url: + url: '{{ test_server }}custom/collections/namespace3-name-1.0.0.tar.gz' + dest: '{{ galaxy_dir }}/namespace3.tar.gz' + +- name: install a collection from a tarball - {{ test_name }} + command: ansible-galaxy collection install '{{ galaxy_dir }}/namespace3.tar.gz' + register: install_tarball + environment: + ANSIBLE_COLLECTIONS_PATHS: '{{ galaxy_dir }}/ansible_collections' + +- name: get result of install collection from a tarball - {{ test_name }} + slurp: + path: '{{ galaxy_dir }}/ansible_collections/namespace3/name/MANIFEST.json' + register: install_tarball_actual + +- name: assert install a collection from a tarball - {{ test_name }} + assert: + that: + - '"Installing ''namespace3.name:1.0.0'' to" in install_tarball.stdout' + - (install_tarball_actual.content | b64decode | from_json).collection_info.version == '1.0.0' + +- name: install a collection from a URI - {{ test_name }} + command: ansible-galaxy collection install '{{ test_server }}custom/collections/namespace4-name-1.0.0.tar.gz' + register: install_uri + environment: + ANSIBLE_COLLECTIONS_PATHS: '{{ galaxy_dir }}/ansible_collections' + +- name: get result of install collection from a URI - {{ test_name }} + slurp: + path: '{{ galaxy_dir }}/ansible_collections/namespace4/name/MANIFEST.json' + register: install_uri_actual + +- name: assert install a collection from a URI - {{ test_name }} + assert: + that: + - '"Installing ''namespace4.name:1.0.0'' to" in install_uri.stdout' + - (install_uri_actual.content | b64decode | from_json).collection_info.version == '1.0.0' + +- name: fail to install a collection with an undefined URL - {{ test_name }} + command: ansible-galaxy collection install namespace5.name + register: fail_undefined_server + failed_when: '"No setting was provided for required configuration plugin_type: galaxy_server plugin: undefined" not in fail_undefined_server.stderr' + environment: + ANSIBLE_GALAXY_SERVER_LIST: undefined + +- name: install a collection with an empty server list - {{ test_name }} + command: ansible-galaxy collection install namespace5.name -s '{{ test_server }}' + register: install_empty_server_list + environment: + ANSIBLE_COLLECTIONS_PATHS: '{{ galaxy_dir }}/ansible_collections' + ANSIBLE_GALAXY_SERVER_LIST: '' + +- name: get result of a collection with an empty server list - {{ test_name }} + slurp: + path: '{{ galaxy_dir }}/ansible_collections/namespace5/name/MANIFEST.json' + register: install_empty_server_list_actual + +- name: assert install a collection with an empty server list - {{ test_name }} + assert: + that: + - '"Installing ''namespace5.name:1.0.0'' to" in install_empty_server_list.stdout' + - (install_empty_server_list_actual.content | b64decode | from_json).collection_info.version == '1.0.0' + +- name: remove test collection install directory - {{ test_name }} + file: + path: '{{ galaxy_dir }}/ansible_collections' + state: absent diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/main.yml b/test/integration/targets/ansible-galaxy-collection/tasks/main.yml new file mode 100644 index 0000000000..6dc2ab1c01 --- /dev/null +++ b/test/integration/targets/ansible-galaxy-collection/tasks/main.yml @@ -0,0 +1,150 @@ +--- +- name: set some facts for tests + set_fact: + galaxy_dir: "{{ remote_tmp_dir }}/galaxy" + +- name: create scratch dir used for testing + file: + path: '{{ galaxy_dir }}/scratch' + state: directory + +- name: run ansible-galaxy collection init tests + import_tasks: init.yml + +- name: run ansible-galaxy collection build tests + import_tasks: build.yml + +- name: create test ansible.cfg that contains the Galaxy server list + template: + src: ansible.cfg.j2 + dest: '{{ galaxy_dir }}/ansible.cfg' + +- name: run ansible-galaxy collection publish tests for {{ test_name }} + include_tasks: publish.yml + vars: + test_name: '{{ item.name }}' + test_server: '{{ item.server }}' + with_items: + - name: galaxy + server: '{{ fallaxy_galaxy_server }}' + - name: automation_hub + server: '{{ fallaxy_ah_server }}' + +# We use a module for this so we can speed up the test time. +- name: setup test collections for install test + setup_collections: + server: '{{ fallaxy_galaxy_server }}' + token: '{{ fallaxy_token }}' + collections: + # Scenario to test out pre-release being ignored unless explicitly set and version pagination. + - namespace: namespace1 + name: name1 + version: 0.0.1 + - namespace: namespace1 + name: name1 + version: 0.0.2 + - namespace: namespace1 + name: name1 + version: 0.0.3 + - namespace: namespace1 + name: name1 + version: 0.0.4 + - namespace: namespace1 + name: name1 + version: 0.0.5 + - namespace: namespace1 + name: name1 + version: 0.0.6 + - namespace: namespace1 + name: name1 + version: 0.0.7 + - namespace: namespace1 + name: name1 + version: 0.0.8 + - namespace: namespace1 + name: name1 + version: 0.0.9 + - namespace: namespace1 + name: name1 + version: 0.0.10 + - namespace: namespace1 + name: name1 + version: 0.1.0 + - namespace: namespace1 + name: name1 + version: 1.0.0 + - namespace: namespace1 + name: name1 + version: 1.0.9 + - namespace: namespace1 + name: name1 + version: 1.1.0-beta.1 + + # Pad out number of namespaces for pagination testing + - namespace: namespace2 + name: name + - namespace: namespace3 + name: name + - namespace: namespace4 + name: name + - namespace: namespace5 + name: name + - namespace: namespace6 + name: name + - namespace: namespace7 + name: name + - namespace: namespace8 + name: name + - namespace: namespace9 + name: name + + # Complex dependency resolution + - namespace: parent_dep + name: parent_collection + dependencies: + child_dep.child_collection: '>=0.5.0,<1.0.0' + - namespace: child_dep + name: child_collection + version: 0.4.0 + - namespace: child_dep + name: child_collection + version: 0.5.0 + - namespace: child_dep + name: child_collection + version: 0.9.9 + dependencies: + child_dep.child_dep2: '!=1.2.3' + - namespace: child_dep + name: child_collection + - namespace: child_dep + name: child_dep2 + version: 1.2.2 + - namespace: child_dep + name: child_dep2 + version: 1.2.3 + + # Dep resolution failure + - namespace: fail_namespace + name: fail_collection + version: 2.1.2 + dependencies: + fail_dep.name: '0.0.5' + fail_dep2.name: '<0.0.5' + - namespace: fail_dep + name: name + version: '0.0.5' + dependencies: + fail_dep2.name: '>0.0.5' + - namespace: fail_dep2 + name: name + +- name: run ansible-galaxy collection install tests for {{ test_name }} + include_tasks: install.yml + vars: + test_name: '{{ item.name }}' + test_server: '{{ item.server }}' + with_items: + - name: galaxy + server: '{{ fallaxy_galaxy_server }}' + - name: automation_hub + server: '{{ fallaxy_ah_server }}' diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/publish.yml b/test/integration/targets/ansible-galaxy-collection/tasks/publish.yml new file mode 100644 index 0000000000..056d29cb28 --- /dev/null +++ b/test/integration/targets/ansible-galaxy-collection/tasks/publish.yml @@ -0,0 +1,46 @@ +--- +- name: fail to publish with no token - {{ test_name }} + command: ansible-galaxy collection publish ansible_test-my_collection-1.0.0.tar.gz -s {{ test_server }} + args: + chdir: '{{ galaxy_dir }}' + register: fail_no_token + failed_when: '"HTTP Code: 401" not in fail_no_token.stderr' + +- name: fail to publish with invalid token - {{ test_name }} + command: ansible-galaxy collection publish ansible_test-my_collection-1.0.0.tar.gz -s {{ test_server }} --api-key fail + args: + chdir: '{{ galaxy_dir }}' + register: fail_invalid_token + failed_when: '"HTTP Code: 401" not in fail_invalid_token.stderr' + +- name: publish collection - {{ test_name }} + command: ansible-galaxy collection publish ansible_test-my_collection-1.0.0.tar.gz -s {{ test_server }} --api-key {{ fallaxy_token }} + args: + chdir: '{{ galaxy_dir }}' + register: publish_collection + +- name: get result of publish collection - {{ test_name }} + uri: + url: '{{ test_server }}v2/collections/ansible_test/my_collection/versions/1.0.0/' + return_content: yes + register: publish_collection_actual + +- name: assert publish collection - {{ test_name }} + assert: + that: + - '"Collection has been successfully published and imported to the Galaxy server" in publish_collection.stdout' + - publish_collection_actual.json.metadata.name == 'my_collection' + - publish_collection_actual.json.metadata.namespace == 'ansible_test' + - publish_collection_actual.json.metadata.version == '1.0.0' + +- name: fail to publish existing collection version - {{ test_name }} + command: ansible-galaxy collection publish ansible_test-my_collection-1.0.0.tar.gz -s {{ test_server }} --api-key {{ fallaxy_token }} + args: + chdir: '{{ galaxy_dir }}' + register: fail_publish_existing + failed_when: '"Artifact already exists" not in fail_publish_existing.stderr' + +- name: reset published collections - {{ test_name }} + uri: + url: '{{ test_server }}custom/reset/' + method: POST diff --git a/test/integration/targets/ansible-galaxy-collection/templates/ansible.cfg.j2 b/test/integration/targets/ansible-galaxy-collection/templates/ansible.cfg.j2 new file mode 100644 index 0000000000..74d36aacfb --- /dev/null +++ b/test/integration/targets/ansible-galaxy-collection/templates/ansible.cfg.j2 @@ -0,0 +1,10 @@ +[galaxy] +server_list=galaxy,automation_hub + +[galaxy_server.galaxy] +url={{ fallaxy_galaxy_server }} +token={{ fallaxy_token }} + +[galaxy_server.automation_hub] +url={{ fallaxy_ah_server }} +token={{ fallaxy_token }} diff --git a/test/integration/targets/ansible-galaxy/runme.sh b/test/integration/targets/ansible-galaxy/runme.sh index f815eccc2a..546c9aa5cc 100755 --- a/test/integration/targets/ansible-galaxy/runme.sh +++ b/test/integration/targets/ansible-galaxy/runme.sh @@ -108,108 +108,4 @@ EOF popd # ${galaxy_testdir} rm -fr "${galaxy_testdir}" -################################# -# ansible-galaxy collection tests -################################# - -f_ansible_galaxy_status \ - "collection init tests to make sure the relative dir logic works" -galaxy_testdir=$(mktemp -d) -pushd "${galaxy_testdir}" - - ansible-galaxy collection init ansible_test.my_collection "$@" - - # Test that the collection skeleton was created in the expected directory - for galaxy_collection_dir in "docs" "plugins" "roles" - do - [[ -d "${galaxy_testdir}/ansible_test/my_collection/${galaxy_collection_dir}" ]] - done - -popd # ${galaxy_testdir} -rm -fr "${galaxy_testdir}" - -f_ansible_galaxy_status \ - "collection init tests to make sure the --init-path logic works" -galaxy_testdir=$(mktemp -d) -pushd "${galaxy_testdir}" - - ansible-galaxy collection init ansible_test.my_collection --init-path "${galaxy_testdir}/test" "$@" - - # Test that the collection skeleton was created in the expected directory - for galaxy_collection_dir in "docs" "plugins" "roles" - do - [[ -d "${galaxy_testdir}/test/ansible_test/my_collection/${galaxy_collection_dir}" ]] - done - -popd # ${galaxy_testdir} - -f_ansible_galaxy_status \ - "collection build test creating artifact in current directory" - -pushd "${galaxy_testdir}/test/ansible_test/my_collection" - - ansible-galaxy collection build "$@" - - [[ -f "${galaxy_testdir}/test/ansible_test/my_collection/ansible_test-my_collection-1.0.0.tar.gz" ]] - -popd # ${galaxy_testdir}/ansible_test/my_collection - -f_ansible_galaxy_status \ - "collection build test to make sure we can specify a relative path" - -pushd "${galaxy_testdir}" - - ansible-galaxy collection build "test/ansible_test/my_collection" "$@" - - [[ -f "${galaxy_testdir}/ansible_test-my_collection-1.0.0.tar.gz" ]] - - # Make sure --force works - ansible-galaxy collection build "test/ansible_test/my_collection" --force "$@" - - [[ -f "${galaxy_testdir}/ansible_test-my_collection-1.0.0.tar.gz" ]] - -f_ansible_galaxy_status \ - "collection install from local tarball test" - - ansible-galaxy collection install "ansible_test-my_collection-1.0.0.tar.gz" -p ./install "$@" | tee out.txt - - [[ -f "${galaxy_testdir}/install/ansible_collections/ansible_test/my_collection/MANIFEST.json" ]] - grep "Installing 'ansible_test.my_collection:1.0.0' to .*" out.txt - - -f_ansible_galaxy_status \ - "collection install with existing collection and without --force" - - ansible-galaxy collection install "ansible_test-my_collection-1.0.0.tar.gz" -p ./install "$@" | tee out.txt - - [[ -f "${galaxy_testdir}/install/ansible_collections/ansible_test/my_collection/MANIFEST.json" ]] - grep "Skipping 'ansible_test.my_collection' as it is already installed" out.txt - -f_ansible_galaxy_status \ - "collection install with existing collection and with --force" - - ansible-galaxy collection install "ansible_test-my_collection-1.0.0.tar.gz" -p ./install --force "$@" | tee out.txt - - [[ -f "${galaxy_testdir}/install/ansible_collections/ansible_test/my_collection/MANIFEST.json" ]] - grep "Installing 'ansible_test.my_collection:1.0.0' to .*" out.txt - -f_ansible_galaxy_status \ - "ansible-galaxy with a sever list with an undefined URL" - - ANSIBLE_GALAXY_SERVER_LIST=undefined ansible-galaxy collection install "ansible_test-my_collection-1.0.0.tar.gz" -p ./install --force "$@" 2>&1 | tee out.txt || echo "expected failure" - - grep "No setting was provided for required configuration plugin_type: galaxy_server plugin: undefined setting: url" out.txt - -f_ansible_galaxy_status \ - "ansible-galaxy with an empty server list" - - ANSIBLE_GALAXY_SERVER_LIST='' ansible-galaxy collection install "ansible_test-my_collection-1.0.0.tar.gz" -p ./install --force "$@" | tee out.txt - - [[ -f "${galaxy_testdir}/install/ansible_collections/ansible_test/my_collection/MANIFEST.json" ]] - grep "Installing 'ansible_test.my_collection:1.0.0' to .*" out.txt - -popd # ${galaxy_testdir} - -rm -fr "${galaxy_testdir}" - rm -fr "${galaxy_local_test_role_dir}" diff --git a/test/lib/ansible_test/_internal/cloud/fallaxy.py b/test/lib/ansible_test/_internal/cloud/fallaxy.py new file mode 100644 index 0000000000..989797f8dc --- /dev/null +++ b/test/lib/ansible_test/_internal/cloud/fallaxy.py @@ -0,0 +1,177 @@ +"""Fallaxy (ansible-galaxy) plugin for integration tests.""" +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import os +import uuid + +from . import ( + CloudProvider, + CloudEnvironment, + CloudEnvironmentConfig, +) + +from ..util import ( + find_executable, + display, +) + +from ..docker_util import ( + docker_run, + docker_rm, + docker_inspect, + docker_pull, + get_docker_container_id, +) + + +class FallaxyProvider(CloudProvider): + """Fallaxy plugin. + + Sets up Fallaxy (ansible-galaxy) stub server for tests. + + It's source source itself resides at: https://github.com/ansible/fallaxy-test-container + """ + + DOCKER_SIMULATOR_NAME = 'fallaxy-stub' + + def __init__(self, args): + """ + :type args: TestConfig + """ + super(FallaxyProvider, self).__init__(args) + + if os.environ.get('ANSIBLE_FALLAXY_CONTAINER'): + self.image = os.environ.get('ANSIBLE_FALLAXY_CONTAINER') + else: + self.image = 'quay.io/ansible/fallaxy-test-container:1.0.0' + self.container_name = '' + + def filter(self, targets, exclude): + """Filter out the tests with the necessary config and res unavailable. + + :type targets: tuple[TestTarget] + :type exclude: list[str] + """ + docker_cmd = 'docker' + docker = find_executable(docker_cmd, required=False) + + if docker: + return + + skip = 'cloud/%s/' % self.platform + skipped = [target.name for target in targets if skip in target.aliases] + + if skipped: + exclude.append(skip) + display.warning('Excluding tests marked "%s" which require the "%s" command: %s' + % (skip.rstrip('/'), docker_cmd, ', '.join(skipped))) + + def setup(self): + """Setup cloud resource before delegation and reg cleanup callback.""" + super(FallaxyProvider, self).setup() + + if self._use_static_config(): + self._setup_static() + else: + self._setup_dynamic() + + def get_docker_run_options(self): + """Get additional options needed when delegating tests to a container. + + :rtype: list[str] + """ + return ['--link', self.DOCKER_SIMULATOR_NAME] if self.managed else [] + + def cleanup(self): + """Clean up the resource and temporary configs files after tests.""" + if self.container_name: + docker_rm(self.args, self.container_name) + + super(FallaxyProvider, self).cleanup() + + def _setup_dynamic(self): + container_id = get_docker_container_id() + + if container_id: + display.info('Running in docker container: %s' % container_id, verbosity=1) + + self.container_name = self.DOCKER_SIMULATOR_NAME + + results = docker_inspect(self.args, self.container_name) + + if results and not results[0].get('State', {}).get('Running'): + docker_rm(self.args, self.container_name) + results = [] + + display.info('%s Fallaxy simulator docker container.' + % ('Using the existing' if results else 'Starting a new'), + verbosity=1) + + fallaxy_port = 8080 + fallaxy_token = str(uuid.uuid4()).replace('-', '') + + if not results: + if self.args.docker or container_id: + publish_ports = [] + else: + # publish the simulator ports when not running inside docker + publish_ports = [ + '-p', ':'.join((str(fallaxy_port),) * 2), + ] + + if not os.environ.get('ANSIBLE_FALLAXY_CONTAINER'): + docker_pull(self.args, self.image) + + docker_run( + self.args, + self.image, + ['-d', '--name', self.container_name, '-e', 'FALLAXY_TOKEN=%s' % fallaxy_token] + publish_ports, + ) + + if self.args.docker: + fallaxy_host = self.DOCKER_SIMULATOR_NAME + elif container_id: + fallaxy_host = self._get_simulator_address() + display.info('Found Fallaxy simulator container address: %s' % fallaxy_host, verbosity=1) + else: + fallaxy_host = 'localhost' + + self._set_cloud_config('FALLAXY_HOST', fallaxy_host) + self._set_cloud_config('FALLAXY_PORT', str(fallaxy_port)) + self._set_cloud_config('FALLAXY_TOKEN', fallaxy_token) + + def _get_simulator_address(self): + results = docker_inspect(self.args, self.container_name) + ipaddress = results[0]['NetworkSettings']['IPAddress'] + return ipaddress + + def _setup_static(self): + raise NotImplementedError() + + +class FallaxyEnvironment(CloudEnvironment): + """Fallaxy environment plugin. + + Updates integration test environment after delegation. + """ + def get_environment_config(self): + """ + :rtype: CloudEnvironmentConfig + """ + fallaxy_token = self._get_cloud_config('FALLAXY_TOKEN') + fallaxy_host = self._get_cloud_config('FALLAXY_HOST') + fallaxy_port = self._get_cloud_config('FALLAXY_PORT') + + return CloudEnvironmentConfig( + ansible_vars=dict( + fallaxy_token=fallaxy_token, + fallaxy_galaxy_server='http://%s:%s/api/' % (fallaxy_host, fallaxy_port), + fallaxy_ah_server='http://%s:%s/api/automation-hub/' % (fallaxy_host, fallaxy_port), + ), + env_vars=dict( + FALLAXY_TOKEN=fallaxy_token, + FALLAXY_GALAXY_SERVER='http://%s:%s/api/' % (fallaxy_host, fallaxy_port), + FALLAXY_AH_SERVER='http://%s:%s/api/automation-hub/' % (fallaxy_host, fallaxy_port), + ), + ) diff --git a/test/units/galaxy/test_api.py b/test/units/galaxy/test_api.py index 78b82b9f08..8f46b3efad 100644 --- a/test/units/galaxy/test_api.py +++ b/test/units/galaxy/test_api.py @@ -341,7 +341,7 @@ def test_publish_failure(api_version, collection_url, response, expected, collec @pytest.mark.parametrize('server_url, api_version, token_type, token_ins, import_uri, full_import_uri', [ ('https://galaxy.server.com/api', 'v2', 'Token', GalaxyToken('my token'), '1234', - 'https://galaxy.server.com/api/v2/collection-imports/1234'), + 'https://galaxy.server.com/api/v2/collection-imports/1234/'), ('https://galaxy.server.com/api/automation-hub/', 'v3', 'Bearer', KeycloakToken(auth_url='https://api.test/'), '1234', 'https://galaxy.server.com/api/automation-hub/v3/imports/collections/1234/'), @@ -374,7 +374,7 @@ def test_wait_import_task(server_url, api_version, token_type, token_ins, import @pytest.mark.parametrize('server_url, api_version, token_type, token_ins, import_uri, full_import_uri', [ ('https://galaxy.server.com/api/', 'v2', 'Token', GalaxyToken('my token'), '1234', - 'https://galaxy.server.com/api/v2/collection-imports/1234'), + 'https://galaxy.server.com/api/v2/collection-imports/1234/'), ('https://galaxy.server.com/api/automation-hub', 'v3', 'Bearer', KeycloakToken(auth_url='https://api.test/'), '1234', 'https://galaxy.server.com/api/automation-hub/v3/imports/collections/1234/'), @@ -421,7 +421,7 @@ def test_wait_import_task_multiple_requests(server_url, api_version, token_type, @pytest.mark.parametrize('server_url, api_version, token_type, token_ins, import_uri, full_import_uri,', [ ('https://galaxy.server.com/api/', 'v2', 'Token', GalaxyToken('my token'), '1234', - 'https://galaxy.server.com/api/v2/collection-imports/1234'), + 'https://galaxy.server.com/api/v2/collection-imports/1234/'), ('https://galaxy.server.com/api/automation-hub/', 'v3', 'Bearer', KeycloakToken(auth_url='https://api.test/'), '1234', 'https://galaxy.server.com/api/automation-hub/v3/imports/collections/1234/'), @@ -498,7 +498,7 @@ def test_wait_import_task_with_failure(server_url, api_version, token_type, toke @pytest.mark.parametrize('server_url, api_version, token_type, token_ins, import_uri, full_import_uri', [ ('https://galaxy.server.com/api/', 'v2', 'Token', GalaxyToken('my_token'), '1234', - 'https://galaxy.server.com/api/v2/collection-imports/1234'), + 'https://galaxy.server.com/api/v2/collection-imports/1234/'), ('https://galaxy.server.com/api/automation-hub/', 'v3', 'Bearer', KeycloakToken(auth_url='https://api.test/'), '1234', 'https://galaxy.server.com/api/automation-hub/v3/imports/collections/1234/'), @@ -571,7 +571,7 @@ def test_wait_import_task_with_failure_no_error(server_url, api_version, token_t @pytest.mark.parametrize('server_url, api_version, token_type, token_ins, import_uri, full_import_uri', [ ('https://galaxy.server.com/api', 'v2', 'Token', GalaxyToken('my token'), '1234', - 'https://galaxy.server.com/api/v2/collection-imports/1234'), + 'https://galaxy.server.com/api/v2/collection-imports/1234/'), ('https://galaxy.server.com/api/automation-hub', 'v3', 'Bearer', KeycloakToken(auth_url='https://api.test/'), '1234', 'https://galaxy.server.com/api/automation-hub/v3/imports/collections/1234/'), @@ -787,34 +787,34 @@ def test_get_collection_versions(api_version, token_type, token_ins, response, m { 'count': 6, 'links': { - 'next': 'https://galaxy.server.com/api/v3/collections/namespace/collection/versions/?page=2', + 'next': '/api/v3/collections/namespace/collection/versions/?page=2', 'previous': None, }, 'data': [ { 'version': '1.0.0', - 'href': 'https://galaxy.server.com/api/v3/collections/namespace/collection/versions/1.0.0', + 'href': '/api/v3/collections/namespace/collection/versions/1.0.0', }, { 'version': '1.0.1', - 'href': 'https://galaxy.server.com/api/v3/collections/namespace/collection/versions/1.0.1', + 'href': '/api/v3/collections/namespace/collection/versions/1.0.1', }, ], }, { 'count': 6, 'links': { - 'next': 'https://galaxy.server.com/api/v3/collections/namespace/collection/versions/?page=3', - 'previous': 'https://galaxy.server.com/api/v3/collections/namespace/collection/versions', + 'next': '/api/v3/collections/namespace/collection/versions/?page=3', + 'previous': '/api/v3/collections/namespace/collection/versions', }, 'data': [ { 'version': '1.0.2', - 'href': 'https://galaxy.server.com/api/v3/collections/namespace/collection/versions/1.0.2', + 'href': '/api/v3/collections/namespace/collection/versions/1.0.2', }, { 'version': '1.0.3', - 'href': 'https://galaxy.server.com/api/v3/collections/namespace/collection/versions/1.0.3', + 'href': '/api/v3/collections/namespace/collection/versions/1.0.3', }, ], }, @@ -822,16 +822,16 @@ def test_get_collection_versions(api_version, token_type, token_ins, response, m 'count': 6, 'links': { 'next': None, - 'previous': 'https://galaxy.server.com/api/v3/collections/namespace/collection/versions/?page=2', + 'previous': '/api/v3/collections/namespace/collection/versions/?page=2', }, 'data': [ { 'version': '1.0.4', - 'href': 'https://galaxy.server.com/api/v3/collections/namespace/collection/versions/1.0.4', + 'href': '/api/v3/collections/namespace/collection/versions/1.0.4', }, { 'version': '1.0.5', - 'href': 'https://galaxy.server.com/api/v3/collections/namespace/collection/versions/1.0.5', + 'href': '/api/v3/collections/namespace/collection/versions/1.0.5', }, ], }, |