diff options
author | Yanis Guenane <yguenane@redhat.com> | 2020-04-16 01:22:17 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-04-15 16:22:17 -0700 |
commit | 46d82179d8fb32ec51ac0ba03ce4445d17c13d34 (patch) | |
tree | 2538f6d4364b0815b0341eaa4c903d3ad702b157 | |
parent | abb807e5ddf49f7308a7609090648b52bf24acff (diff) | |
download | ansible-46d82179d8fb32ec51ac0ba03ce4445d17c13d34.tar.gz |
Testing: Add support for CentOS Linux On Power platform (#68130)
* Testing: Add CentOS Linux On Power platform
* Add arch designation to remotes.
This avoids overloading the provider with the arch.
Also add a changelog entry.
Co-authored-by: Matt Clay <matt@mystile.com>
26 files changed, 187 insertions, 38 deletions
diff --git a/changelogs/fragments/ansible-test-remote-power.yml b/changelogs/fragments/ansible-test-remote-power.yml new file mode 100644 index 0000000000..e4a34e39a9 --- /dev/null +++ b/changelogs/fragments/ansible-test-remote-power.yml @@ -0,0 +1,7 @@ +minor_changes: + - ansible-test now supports the ``--remote power/centos/7`` platform option. + - ansible-test now supports skip aliases in the format ``skip/{platform}/{version}`` for the ``--remote`` option. + This is preferred over the older ``skip/{platform}{version}`` format which included no ``/`` between the platform and version. + - ansible-test now supports skip aliases in the format ``skip/{arch}/{platform}`` and ``skip/{arch}/{platform}/{version}`` where ``arch`` can be ``power``. + These aliases are only effective for the ``--remote`` option. + - ansible-test provides clearer error messages when failing to detect the provider to use with the ``--remote`` option. diff --git a/shippable.yml b/shippable.yml index 392e460bd6..f2c25a4a76 100644 --- a/shippable.yml +++ b/shippable.yml @@ -42,6 +42,7 @@ matrix: - env: T=linux/opensuse15/1 - env: T=linux/ubuntu1604/1 - env: T=linux/ubuntu1804/1 + - env: T=power/centos/7/1 - env: T=aix/7.2/2 - env: T=osx/10.11/2 @@ -58,6 +59,7 @@ matrix: - env: T=linux/opensuse15/2 - env: T=linux/ubuntu1604/2 - env: T=linux/ubuntu1804/2 + - env: T=power/centos/7/2 - env: T=aix/7.2/3 - env: T=osx/10.11/3 @@ -74,6 +76,7 @@ matrix: - env: T=linux/opensuse15/3 - env: T=linux/ubuntu1604/3 - env: T=linux/ubuntu1804/3 + - env: T=power/centos/7/3 - env: T=aix/7.2/4 - env: T=osx/10.11/4 @@ -90,6 +93,7 @@ matrix: - env: T=linux/opensuse15/4 - env: T=linux/ubuntu1604/4 - env: T=linux/ubuntu1804/4 + - env: T=power/centos/7/4 - env: T=aix/7.2/5 - env: T=osx/10.11/5 @@ -106,6 +110,7 @@ matrix: - env: T=linux/opensuse15/5 - env: T=linux/ubuntu1604/5 - env: T=linux/ubuntu1804/5 + - env: T=power/centos/7/5 - env: T=fallaxy/2.7/1 - env: T=fallaxy/3.6/1 @@ -125,6 +130,7 @@ matrix: - env: T=i/linux/opensuse15 - env: T=i/linux/ubuntu1604 - env: T=i/linux/ubuntu1804 + - env: T=i/power/centos/7 - env: T=i/windows/2012 - env: T=i/windows/2012-R2 diff --git a/test/integration/targets/binary_modules_posix/aliases b/test/integration/targets/binary_modules_posix/aliases index 1d8e11e384..e33a31e870 100644 --- a/test/integration/targets/binary_modules_posix/aliases +++ b/test/integration/targets/binary_modules_posix/aliases @@ -1,3 +1,4 @@ shippable/posix/group3 needs/target/binary_modules skip/aix +skip/power/centos diff --git a/test/integration/targets/dnf/aliases b/test/integration/targets/dnf/aliases index e469a11971..802c8cfad5 100644 --- a/test/integration/targets/dnf/aliases +++ b/test/integration/targets/dnf/aliases @@ -1,5 +1,6 @@ destructive shippable/posix/group4 skip/aix +skip/power/centos skip/freebsd skip/osx diff --git a/test/integration/targets/incidental_consul/aliases b/test/integration/targets/incidental_consul/aliases index 87066db952..0a22af0f92 100644 --- a/test/integration/targets/incidental_consul/aliases +++ b/test/integration/targets/incidental_consul/aliases @@ -1,3 +1,4 @@ shippable/posix/incidental destructive skip/aix +skip/power/centos diff --git a/test/integration/targets/incidental_inventory_docker_swarm/aliases b/test/integration/targets/incidental_inventory_docker_swarm/aliases index cdb5961501..cbebbc7dfe 100644 --- a/test/integration/targets/incidental_inventory_docker_swarm/aliases +++ b/test/integration/targets/incidental_inventory_docker_swarm/aliases @@ -1,5 +1,6 @@ shippable/posix/incidental skip/aix +skip/power/centos skip/osx skip/freebsd destructive diff --git a/test/integration/targets/incidental_lookup_hashi_vault/aliases b/test/integration/targets/incidental_lookup_hashi_vault/aliases index 7e29c80ee2..3dded09a49 100644 --- a/test/integration/targets/incidental_lookup_hashi_vault/aliases +++ b/test/integration/targets/incidental_lookup_hashi_vault/aliases @@ -3,4 +3,5 @@ destructive needs/target/incidental_setup_openssl needs/file/test/lib/ansible_test/_data/requirements/constraints.txt skip/aix +skip/power/centos skip/python2.6 diff --git a/test/integration/targets/incidental_ufw/aliases b/test/integration/targets/incidental_ufw/aliases index ff7ad7ce4a..0013b22964 100644 --- a/test/integration/targets/incidental_ufw/aliases +++ b/test/integration/targets/incidental_ufw/aliases @@ -1,5 +1,6 @@ shippable/posix/incidental skip/aix +skip/power/centos skip/osx skip/freebsd skip/rhel8.0 diff --git a/test/integration/targets/incidental_xml/aliases b/test/integration/targets/incidental_xml/aliases index b9ab365826..fc0963c1e3 100644 --- a/test/integration/targets/incidental_xml/aliases +++ b/test/integration/targets/incidental_xml/aliases @@ -1,3 +1,4 @@ destructive shippable/posix/incidental skip/aix +skip/power/centos diff --git a/test/integration/targets/package/aliases b/test/integration/targets/package/aliases index 0b484bbab6..ae0bf1deea 100644 --- a/test/integration/targets/package/aliases +++ b/test/integration/targets/package/aliases @@ -1,3 +1,4 @@ shippable/posix/group1 destructive skip/aix +skip/power/centos diff --git a/test/integration/targets/pip/aliases b/test/integration/targets/pip/aliases index 8d8cc50ef8..ac6aa40434 100644 --- a/test/integration/targets/pip/aliases +++ b/test/integration/targets/pip/aliases @@ -1,3 +1,4 @@ destructive shippable/posix/group5 skip/aix +skip/power/centos diff --git a/test/integration/targets/subversion/aliases b/test/integration/targets/subversion/aliases index d3e8e8020c..25c180e507 100644 --- a/test/integration/targets/subversion/aliases +++ b/test/integration/targets/subversion/aliases @@ -1,6 +1,7 @@ setup/always/setup_passlib shippable/posix/group2 skip/aix +skip/power/centos skip/osx destructive needs/root diff --git a/test/integration/targets/unarchive/aliases b/test/integration/targets/unarchive/aliases index db9bbd8c42..559c7606c9 100644 --- a/test/integration/targets/unarchive/aliases +++ b/test/integration/targets/unarchive/aliases @@ -2,3 +2,4 @@ needs/root shippable/posix/group2 destructive skip/aix +skip/power/centos diff --git a/test/integration/targets/yum/aliases b/test/integration/targets/yum/aliases index e469a11971..802c8cfad5 100644 --- a/test/integration/targets/yum/aliases +++ b/test/integration/targets/yum/aliases @@ -1,5 +1,6 @@ destructive shippable/posix/group4 skip/aix +skip/power/centos skip/freebsd skip/osx diff --git a/test/integration/targets/yum_repository/aliases b/test/integration/targets/yum_repository/aliases index 0b484bbab6..ae0bf1deea 100644 --- a/test/integration/targets/yum_repository/aliases +++ b/test/integration/targets/yum_repository/aliases @@ -1,3 +1,4 @@ shippable/posix/group1 destructive skip/aix +skip/power/centos diff --git a/test/lib/ansible_test/_data/completion/remote.txt b/test/lib/ansible_test/_data/completion/remote.txt index 7e3327f21f..36bced6624 100644 --- a/test/lib/ansible_test/_data/completion/remote.txt +++ b/test/lib/ansible_test/_data/completion/remote.txt @@ -5,3 +5,4 @@ rhel/7.6 python=2.7 rhel/7.8 python=2.7 rhel/8.1 python=3.6 aix/7.2 python=2.7 httptester=disabled temp-unicode=disabled pip-check=disabled +power/centos/7 python=2.7 diff --git a/test/lib/ansible_test/_data/setup/remote.sh b/test/lib/ansible_test/_data/setup/remote.sh index 8a4d6c2d35..6a70863a0e 100644 --- a/test/lib/ansible_test/_data/setup/remote.sh +++ b/test/lib/ansible_test/_data/setup/remote.sh @@ -70,6 +70,21 @@ elif [ "${platform}" = "rhel" ]; then install_pip fi +elif [ "${platform}" = "centos" ]; then + while true; do + yum install -q -y \ + gcc \ + python-devel \ + python-virtualenv \ + python2-cryptography \ + libffi-devel \ + openssl-devel \ + && break + echo "Failed to install packages. Sleeping before trying again..." + sleep 10 + done + + install_pip elif [ "${platform}" = "osx" ]; then while true; do pip install --disable-pip-version-check --quiet \ diff --git a/test/lib/ansible_test/_internal/cli.py b/test/lib/ansible_test/_internal/cli.py index 0c8345718b..f692f944f7 100644 --- a/test/lib/ansible_test/_internal/cli.py +++ b/test/lib/ansible_test/_internal/cli.py @@ -950,7 +950,7 @@ def add_environments(parser, isolated_delegation=True): remote.add_argument('--remote-provider', metavar='PROVIDER', help='remote provider to use: %(choices)s', - choices=['default', 'aws', 'azure', 'parallels'], + choices=['default', 'aws', 'azure', 'parallels', 'ibmvpc', 'ibmps'], default='default') remote.add_argument('--remote-aws-region', diff --git a/test/lib/ansible_test/_internal/config.py b/test/lib/ansible_test/_internal/config.py index 4e3eb7be87..08285553a4 100644 --- a/test/lib/ansible_test/_internal/config.py +++ b/test/lib/ansible_test/_internal/config.py @@ -35,6 +35,29 @@ except AttributeError: TIntegrationConfig = None # pylint: disable=invalid-name +class ParsedRemote: + """A parsed version of a "remote" string.""" + def __init__(self, arch, platform, version): # type: (t.Optional[str], str, str) -> None + self.arch = arch + self.platform = platform + self.version = version + + @staticmethod + def parse(value): # type: (str) -> t.Optional['ParsedRemote'] + """Return a ParsedRemote from the given value or None if the syntax is invalid.""" + parts = value.split('/') + + if len(parts) == 2: + arch = None + platform, version = parts + elif len(parts) == 3: + arch, platform, version = parts + else: + return None + + return ParsedRemote(arch, platform, version) + + class EnvironmentConfig(CommonConfig): """Configuration common to all commands which execute in an environment.""" def __init__(self, args, command): @@ -54,6 +77,14 @@ class EnvironmentConfig(CommonConfig): self.docker_raw = args.docker # type: str self.remote = args.remote # type: str + if self.remote: + self.parsed_remote = ParsedRemote.parse(self.remote) + + if not self.parsed_remote or not self.parsed_remote.platform or not self.parsed_remote.version: + raise ApplicationError('Unrecognized remote "%s" syntax. Use "platform/version" or "arch/platform/version".' % self.remote) + else: + self.parsed_remote = None + self.docker_privileged = args.docker_privileged if 'docker_privileged' in args else False # type: bool self.docker_pull = args.docker_pull if 'docker_pull' in args else False # type: bool self.docker_keep_git = args.docker_keep_git if 'docker_keep_git' in args else False # type: bool diff --git a/test/lib/ansible_test/_internal/core_ci.py b/test/lib/ansible_test/_internal/core_ci.py index 13b475d58b..58bd76449e 100644 --- a/test/lib/ansible_test/_internal/core_ci.py +++ b/test/lib/ansible_test/_internal/core_ci.py @@ -53,7 +53,7 @@ AWS_ENDPOINTS = { class AnsibleCoreCI: """Client for Ansible Core CI services.""" - def __init__(self, args, platform, version, stage='prod', persist=True, load=True, name=None, provider=None): + def __init__(self, args, platform, version, stage='prod', persist=True, load=True, provider=None, arch=None): """ :type args: EnvironmentConfig :type platform: str @@ -61,9 +61,11 @@ class AnsibleCoreCI: :type stage: str :type persist: bool :type load: bool - :type name: str + :type provider: str | None + :type arch: str | None """ self.args = args + self.arch = arch self.platform = platform self.version = version self.stage = stage @@ -73,7 +75,12 @@ class AnsibleCoreCI: self.endpoint = None self.max_threshold = 1 self.retries = 3 - self.name = name if name else '%s-%s' % (self.platform, self.version) + + if self.arch: + self.name = '%s-%s-%s' % (self.arch, self.platform, self.version) + else: + self.name = '%s-%s' % (self.platform, self.version) + self.ci_key = os.path.expanduser('~/.ansible-core-ci.key') self.resource = 'jobs' @@ -98,32 +105,56 @@ class AnsibleCoreCI: 'aix', 'ibmi', ), + ibmvpc=( + 'centos arch=power', # avoid ibmvpc as default for no-arch centos to avoid making centos default to power + ), parallels=( 'osx', ), vmware=( - 'vmware' + 'vmware', ), ) + # Currently ansible-core-ci has no concept of arch selection. This effectively means each provider only supports one arch. + # The list below identifies which platforms accept an arch, and which one. These platforms can only be used with the specified arch. + provider_arches = dict( + ibmvpc='power', + ) + if provider: # override default provider selection (not all combinations are valid) self.provider = provider else: + self.provider = None + for candidate in providers: - if platform in providers[candidate]: + choices = [ + platform, + '%s arch=%s' % (platform, arch), + ] + + if any(choice in providers[candidate] for choice in choices): # assign default provider based on platform self.provider = candidate break - for candidate in providers: - if '%s/%s' % (platform, version) in providers[candidate]: - # assign default provider based on platform and version - self.provider = candidate - break + + # If a provider has been selected, make sure the correct arch (or none) has been selected. + if self.provider: + required_arch = provider_arches.get(self.provider) + + if self.arch != required_arch: + if required_arch: + if self.arch: + raise ApplicationError('Provider "%s" requires the "%s" arch instead of "%s".' % (self.provider, required_arch, self.arch)) + + raise ApplicationError('Provider "%s" requires the "%s" arch.' % (self.provider, required_arch)) + + raise ApplicationError('Provider "%s" does not support specification of an arch.' % self.provider) self.path = os.path.expanduser('~/.ansible/test/instances/%s-%s-%s' % (self.name, self.provider, self.stage)) - if self.provider in ('aws', 'azure', 'ibmps'): + if self.provider in ('aws', 'azure', 'ibmps', 'ibmvpc'): if self.provider != 'aws': self.resource = self.provider @@ -167,7 +198,10 @@ class AnsibleCoreCI: self.endpoints = ['https://access.ws.testing.ansible.com'] self.max_threshold = 1 else: - raise ApplicationError('Unsupported platform: %s' % platform) + if self.arch: + raise ApplicationError('Provider not detected for platform "%s" on arch "%s".' % (self.platform, self.arch)) + + raise ApplicationError('Provider not detected for platform "%s" with no arch specified.' % self.platform) if persist and load and self._load(): try: diff --git a/test/lib/ansible_test/_internal/delegation.py b/test/lib/ansible_test/_internal/delegation.py index 6cae52f651..6f80ee1f6e 100644 --- a/test/lib/ansible_test/_internal/delegation.py +++ b/test/lib/ansible_test/_internal/delegation.py @@ -389,12 +389,9 @@ def delegate_remote(args, exclude, require, integration_targets): :type require: list[str] :type integration_targets: tuple[IntegrationTarget] """ - parts = args.remote.split('/', 1) + remote = args.parsed_remote - platform = parts[0] - version = parts[1] - - core_ci = AnsibleCoreCI(args, platform, version, stage=args.remote_stage, provider=args.remote_provider) + core_ci = AnsibleCoreCI(args, remote.platform, remote.version, stage=args.remote_stage, provider=args.remote_provider, arch=remote.arch) success = False raw = False @@ -422,7 +419,7 @@ def delegate_remote(args, exclude, require, integration_targets): python_version = get_python_version(args, get_remote_completion(), args.remote) - if platform == 'windows': + if remote.platform == 'windows': # Windows doesn't need the ansible-test fluff, just run the SSH command manage = ManageWindowsCI(core_ci) manage.setup(python_version) @@ -457,7 +454,7 @@ def delegate_remote(args, exclude, require, integration_targets): if isinstance(args, TestConfig): if args.coverage and not args.coverage_label: - cmd += ['--coverage-label', 'remote-%s-%s' % (platform, version)] + cmd += ['--coverage-label', 'remote-%s-%s' % (remote.platform, remote.version)] if isinstance(args, IntegrationConfig): if not args.allow_destructive: @@ -479,7 +476,7 @@ def delegate_remote(args, exclude, require, integration_targets): finally: download = False - if platform != 'windows': + if remote.platform != 'windows': download = True if isinstance(args, ShellConfig): @@ -495,7 +492,7 @@ def delegate_remote(args, exclude, require, integration_targets): # AIX cp and GNU cp provide different options, no way could be found to have a common # pattern and achieve the same goal - cp_opts = '-hr' if platform in ['aix', 'ibmi'] else '-a' + cp_opts = '-hr' if remote.platform in ['aix', 'ibmi'] else '-a' manage.ssh('rm -rf {0} && mkdir {0} && cp {1} {2}/* {0}/ && chmod -R a+r {0}'.format(remote_temp_path, cp_opts, remote_results_root)) manage.download(remote_temp_path, local_test_root) diff --git a/test/lib/ansible_test/_internal/executor.py b/test/lib/ansible_test/_internal/executor.py index 9e7e835a51..5105f447fc 100644 --- a/test/lib/ansible_test/_internal/executor.py +++ b/test/lib/ansible_test/_internal/executor.py @@ -1834,27 +1834,29 @@ def get_integration_remote_filter(args, targets): :type targets: tuple[IntegrationTarget] :rtype: list[str] """ - parts = args.remote.split('/', 1) - - platform = parts[0] + remote = args.parsed_remote exclude = [] common_integration_filter(args, targets, exclude) - skip = 'skip/%s/' % platform - skipped = [target.name for target in targets if skip in target.aliases] - if skipped: - exclude.append(skip) - display.warning('Excluding tests marked "%s" which are not supported on %s: %s' - % (skip.rstrip('/'), platform, ', '.join(skipped))) + skips = { + 'skip/%s' % remote.platform: remote.platform, + 'skip/%s/%s' % (remote.platform, remote.version): '%s %s' % (remote.platform, remote.version), + 'skip/%s%s' % (remote.platform, remote.version): '%s %s' % (remote.platform, remote.version), # legacy syntax, use above format + } - skip = 'skip/%s/' % args.remote.replace('/', '') - skipped = [target.name for target in targets if skip in target.aliases] - if skipped: - exclude.append(skip) - display.warning('Excluding tests marked "%s" which are not supported on %s: %s' - % (skip.rstrip('/'), args.remote.replace('/', ' '), ', '.join(skipped))) + if remote.arch: + skips.update({ + 'skip/%s/%s' % (remote.arch, remote.platform): '%s on %s' % (remote.platform, remote.arch), + 'skip/%s/%s/%s' % (remote.arch, remote.platform, remote.version): '%s %s on %s' % (remote.platform, remote.version, remote.arch), + }) + + for skip, description in skips.items(): + skipped = [target.name for target in targets if skip in target.skips] + if skipped: + exclude.append(skip + '/') + display.warning('Excluding tests marked "%s" which are not supported on %s: %s' % (skip, description, ', '.join(skipped))) python_version = get_python_version(args, get_remote_completion(), args.remote) diff --git a/test/lib/ansible_test/_internal/manage_ci.py b/test/lib/ansible_test/_internal/manage_ci.py index 0b5eb5a108..0b77b02b17 100644 --- a/test/lib/ansible_test/_internal/manage_ci.py +++ b/test/lib/ansible_test/_internal/manage_ci.py @@ -213,7 +213,7 @@ class ManagePosixCI: raise NotImplementedError('provider %s has not been implemented' % self.core_ci.provider) elif self.core_ci.platform == 'osx': self.become = ['sudo', '-in', 'PATH=/usr/local/bin:$PATH'] - elif self.core_ci.platform == 'rhel': + elif self.core_ci.platform == 'rhel' or self.core_ci.platform == 'centos': self.become = ['sudo', '-in', 'bash', '-c'] elif self.core_ci.platform in ['aix', 'ibmi']: self.become = [] diff --git a/test/lib/ansible_test/_internal/target.py b/test/lib/ansible_test/_internal/target.py index c3ba81268d..c023424ef6 100644 --- a/test/lib/ansible_test/_internal/target.py +++ b/test/lib/ansible_test/_internal/target.py @@ -638,6 +638,9 @@ class IntegrationTarget(CompletionTarget): targets_relative_path = data_context().content.integration_targets_path + # Collect skip entries before group expansion to avoid registering more specific skip entries as less specific versions. + self.skips = tuple(g for g in groups if g.startswith('skip/')) + # Collect file paths before group expansion to avoid including the directories. # Ignore references to test targets, as those must be defined using `needs/target/*` or other target references. self.needs_file = tuple(sorted(set('/'.join(g.split('/')[2:]) for g in groups if diff --git a/test/utils/shippable/incidental/power.sh b/test/utils/shippable/incidental/power.sh new file mode 100755 index 0000000000..4d2c17c82e --- /dev/null +++ b/test/utils/shippable/incidental/power.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -o pipefail -eux + +declare -a args +IFS='/:' read -ra args <<< "$1" + +platform="${args[1]}" +version="${args[2]}" + +target="shippable/posix/incidental/" + +stage="${S:-prod}" +provider="${P:-default}" + +# shellcheck disable=SC2086 +ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} \ + --remote "power/${platform}/${version}" --remote-terminate always --remote-stage "${stage}" --remote-provider "${provider}" diff --git a/test/utils/shippable/power.sh b/test/utils/shippable/power.sh new file mode 100755 index 0000000000..85003ffc7d --- /dev/null +++ b/test/utils/shippable/power.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -o pipefail -eux + +declare -a args +IFS='/:' read -ra args <<< "$1" + +platform="${args[1]}" +version="${args[2]}" + +if [ "${#args[@]}" -gt 3 ]; then + target="shippable/posix/group${args[3]}/" +else + target="shippable/posix/" +fi + +stage="${S:-prod}" +provider="${P:-default}" + +# shellcheck disable=SC2086 +ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} \ + --remote "power/${platform}/${version}" --remote-terminate always --remote-stage "${stage}" --remote-provider "${provider}" |