diff options
Diffstat (limited to 'test/lib/ansible_test/_util')
50 files changed, 759 insertions, 344 deletions
diff --git a/test/lib/ansible_test/_util/__init__.py b/test/lib/ansible_test/_util/__init__.py new file mode 100644 index 0000000000..d6fc0a8614 --- /dev/null +++ b/test/lib/ansible_test/_util/__init__.py @@ -0,0 +1,3 @@ +"""Nearly empty __init__.py to allow importing under Python 2.x.""" +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type diff --git a/test/lib/ansible_test/_util/controller/cli/ansible_test_cli_stub.py b/test/lib/ansible_test/_util/controller/cli/ansible_test_cli_stub.py deleted file mode 100755 index d12b6334ef..0000000000 --- a/test/lib/ansible_test/_util/controller/cli/ansible_test_cli_stub.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env python -# PYTHON_ARGCOMPLETE_OK -"""Command line entry point for ansible-test.""" - -from __future__ import (absolute_import, division, print_function) -__metaclass__ = type - -import os -import sys - - -def main(): - """Main program entry point.""" - ansible_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - source_root = os.path.join(ansible_root, 'test', 'lib') - - if os.path.exists(os.path.join(source_root, 'ansible_test', '_internal', 'cli.py')): - # running from source, use that version of ansible-test instead of any version that may already be installed - sys.path.insert(0, source_root) - - # noinspection PyProtectedMember - from ansible_test._internal.cli import main as cli_main - - cli_main() - - -if __name__ == '__main__': - main() diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/action-plugin-docs.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/action-plugin-docs.py index 65142e0033..e19b4d98a4 100755..100644 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/action-plugin-docs.py +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/action-plugin-docs.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """Test to verify action plugins have an associated module to provide documentation.""" from __future__ import (absolute_import, division, print_function) __metaclass__ = type diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/changelog.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/changelog.py index 2ccfb24f23..1875ab3aa4 100755..100644 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/changelog.py +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/changelog.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python from __future__ import (absolute_import, division, print_function) __metaclass__ = type diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/empty-init.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/empty-init.py index 8bcd7f9ed9..806c0e6ed1 100755..100644 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/empty-init.py +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/empty-init.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python from __future__ import (absolute_import, division, print_function) __metaclass__ = type diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/future-import-boilerplate.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/future-import-boilerplate.py index 81081eed7b..cdad96551e 100755..100644 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/future-import-boilerplate.py +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/future-import-boilerplate.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python from __future__ import (absolute_import, division, print_function) __metaclass__ = type diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/line-endings.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/line-endings.py index 1e4212d1b8..660b0fce85 100755..100644 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/line-endings.py +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/line-endings.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python from __future__ import (absolute_import, division, print_function) __metaclass__ = type diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/metaclass-boilerplate.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/metaclass-boilerplate.py index 28d06f363b..e3fba1f5dd 100755..100644 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/metaclass-boilerplate.py +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/metaclass-boilerplate.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python from __future__ import (absolute_import, division, print_function) __metaclass__ = type diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-assert.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-assert.py index 78561d966e..d6d710aeff 100755..100644 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-assert.py +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-assert.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python from __future__ import (absolute_import, division, print_function) __metaclass__ = type @@ -10,8 +9,8 @@ ASSERT_RE = re.compile(r'^\s*assert[^a-z0-9_:]') def main(): for path in sys.argv[1:] or sys.stdin.read().splitlines(): - with open(path, 'r') as f: - for i, line in enumerate(f.readlines()): + with open(path, 'r') as file: + for i, line in enumerate(file.readlines()): matches = ASSERT_RE.findall(line) if matches: diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-basestring.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-basestring.py index a35650efad..18a3f6d1d1 100755..100644 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-basestring.py +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-basestring.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python from __future__ import (absolute_import, division, print_function) __metaclass__ = type diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-iteritems.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-iteritems.py index e28b24f4a9..7dfd5b2601 100755..100644 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-iteritems.py +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-iteritems.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python from __future__ import (absolute_import, division, print_function) __metaclass__ = type diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-iterkeys.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-iterkeys.py index 237ee5b1c1..8925e831d2 100755..100644 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-iterkeys.py +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-iterkeys.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python from __future__ import (absolute_import, division, print_function) __metaclass__ = type diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-itervalues.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-itervalues.py index 4bf92ea990..1813415438 100755..100644 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-itervalues.py +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-dict-itervalues.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python from __future__ import (absolute_import, division, print_function) __metaclass__ = type diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-get-exception.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-get-exception.py index c925f5b729..5a267ba0df 100755..100644 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-get-exception.py +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-get-exception.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python from __future__ import (absolute_import, division, print_function) __metaclass__ = type diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-illegal-filenames.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-illegal-filenames.py index 99432ea133..421bbd6229 100755..100644 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-illegal-filenames.py +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-illegal-filenames.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - # a script to check for illegal filenames on various Operating Systems. The # main rules are derived from restrictions on Windows # https://msdn.microsoft.com/en-us/library/aa365247#naming_conventions diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-main-display.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-main-display.py index 74a36ecc58..e5abd64db8 100755..100644 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-main-display.py +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-main-display.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python from __future__ import (absolute_import, division, print_function) __metaclass__ = type @@ -9,8 +8,8 @@ MAIN_DISPLAY_IMPORT = 'from __main__ import display' def main(): for path in sys.argv[1:] or sys.stdin.read().splitlines(): - with open(path, 'r') as f: - for i, line in enumerate(f.readlines()): + with open(path, 'r') as file: + for i, line in enumerate(file.readlines()): if MAIN_DISPLAY_IMPORT in line: lineno = i + 1 colno = line.index(MAIN_DISPLAY_IMPORT) + 1 diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-smart-quotes.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-smart-quotes.py index e44005a55f..8399a36e0b 100755..100644 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-smart-quotes.py +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-smart-quotes.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- from __future__ import (absolute_import, division, print_function) __metaclass__ = type diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-unicode-literals.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-unicode-literals.py index e2201ab106..bb8c8f01d8 100755..100644 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/no-unicode-literals.py +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/no-unicode-literals.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python from __future__ import (absolute_import, division, print_function) __metaclass__ = type diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/replace-urlopen.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/replace-urlopen.py index b2de1ba85d..87575f5189 100755..100644 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/replace-urlopen.py +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/replace-urlopen.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python from __future__ import (absolute_import, division, print_function) __metaclass__ = type diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/runtime-metadata.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/runtime-metadata.py index 7db04ced7b..cad82a5575 100755..100644 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/runtime-metadata.py +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/runtime-metadata.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """Schema validation of ansible-core's ansible_builtin_runtime.yml and collection's meta/runtime.yml""" from __future__ import (absolute_import, division, print_function) __metaclass__ = type @@ -7,7 +6,6 @@ import datetime import os import re import sys -import warnings from functools import partial diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/shebang.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/shebang.py index b945734cf7..6f210651c6 100755..100644 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/shebang.py +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/shebang.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python from __future__ import (absolute_import, division, print_function) __metaclass__ = type @@ -69,9 +68,7 @@ def main(): is_module = True elif re.search('^test/support/[^/]+/collections/ansible_collections/[^/]+/[^/]+/plugins/modules/', path): is_module = True - elif path.startswith('test/lib/ansible_test/_data/'): - pass - elif path.startswith('test/lib/ansible_test/_util/'): + elif path.startswith('test/lib/ansible_test/_util/target/'): pass elif path.startswith('lib/') or path.startswith('test/lib/'): if executable: diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/symlinks.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/symlinks.py index 0585c6b1e5..5603051ac5 100755..100644 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/symlinks.py +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/symlinks.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python from __future__ import (absolute_import, division, print_function) __metaclass__ = type diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/use-argspec-type-path.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/use-argspec-type-path.py index 687136dcdb..68f380b0a9 100755..100644 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/use-argspec-type-path.py +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/use-argspec-type-path.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python from __future__ import (absolute_import, division, print_function) __metaclass__ = type diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/use-compat-six.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/use-compat-six.py index 49cb76c5e2..a8f0b87950 100755..100644 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/use-compat-six.py +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/use-compat-six.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python from __future__ import (absolute_import, division, print_function) __metaclass__ = type diff --git a/test/lib/ansible_test/_util/controller/sanity/pslint/pslint.ps1 b/test/lib/ansible_test/_util/controller/sanity/pslint/pslint.ps1 index 1ef2743acd..9138a29904 100755..100644 --- a/test/lib/ansible_test/_util/controller/sanity/pslint/pslint.ps1 +++ b/test/lib/ansible_test/_util/controller/sanity/pslint/pslint.ps1 @@ -1,4 +1,3 @@ -#!/usr/bin/env pwsh #Requires -Version 6 #Requires -Modules PSScriptAnalyzer, PSSA-PSCustomUseLiteralPath diff --git a/test/lib/ansible_test/_util/controller/sanity/pylint/config/sanity.cfg b/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test-target.cfg index bcf9549fd7..30e40ba1f4 100644 --- a/test/lib/ansible_test/_util/controller/sanity/pylint/config/sanity.cfg +++ b/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test-target.cfg @@ -7,8 +7,8 @@ disable= duplicate-code, # consistent results require running with --jobs 1 and testing all files import-error, # inconsistent results which depend on the availability of imports import-outside-toplevel, # common pattern in ansible related code - missing-docstring, no-name-in-module, # inconsistent results which depend on the availability of imports + no-self-use, raise-missing-from, # Python 2.x does not support raise from super-with-arguments, # Python 2.x does not support super without arguments too-few-public-methods, @@ -21,7 +21,7 @@ disable= too-many-nested-blocks, too-many-return-statements, too-many-statements, - unused-import, # pylint does not understand PEP 484 type hints + useless-return, # complains about returning None when the return type is optional [BASIC] @@ -37,17 +37,16 @@ bad-names= good-names= __metaclass__, C, - e, ex, - f, i, j, k, Run, -module-rgx=[a-z_][a-z0-9_-]{2,40}$ -method-rgx=[a-z_][a-z0-9_]{2,40}$ -function-rgx=[a-z_][a-z0-9_]{2,40}$ +class-attribute-rgx=[A-Za-z_][A-Za-z0-9_]{1,40}$ +attr-rgx=[a-z_][a-z0-9_]{1,40}$ +method-rgx=[a-z_][a-z0-9_]{1,40}$ +function-rgx=[a-z_][a-z0-9_]{1,40}$ [IMPORTS] diff --git a/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test.cfg b/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test.cfg index 187758f409..3c60aa77fe 100644 --- a/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test.cfg +++ b/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test.cfg @@ -1,8 +1,6 @@ [MESSAGES CONTROL] disable= - consider-using-dict-comprehension, # requires Python 2.7+, but we still require Python 2.6 support - consider-using-set-comprehension, # requires Python 2.7+, but we still require Python 2.6 support cyclic-import, # consistent results require running with --jobs 1 and testing all files duplicate-code, # consistent results require running with --jobs 1 and testing all files import-error, # inconsistent results which depend on the availability of imports @@ -10,9 +8,7 @@ disable= no-name-in-module, # inconsistent results which depend on the availability of imports no-self-use, raise-missing-from, # Python 2.x does not support raise from - super-with-arguments, # Python 2.x does not support super without arguments too-few-public-methods, - too-many-ancestors, # inconsistent results between python 3.6 and 3.7+ too-many-arguments, too-many-branches, too-many-instance-attributes, @@ -21,7 +17,7 @@ disable= too-many-nested-blocks, too-many-return-statements, too-many-statements, - unused-import, # pylint does not understand PEP 484 type hints + useless-return, # complains about returning None when the return type is optional [BASIC] diff --git a/test/lib/ansible_test/_util/controller/sanity/pylint/config/code-smell.cfg b/test/lib/ansible_test/_util/controller/sanity/pylint/config/code-smell.cfg new file mode 100644 index 0000000000..739d37576d --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/pylint/config/code-smell.cfg @@ -0,0 +1,55 @@ +[MESSAGES CONTROL] + +disable= + cyclic-import, # consistent results require running with --jobs 1 and testing all files + duplicate-code, # consistent results require running with --jobs 1 and testing all files + import-error, # inconsistent results which depend on the availability of imports + import-outside-toplevel, # common pattern in ansible related code + no-name-in-module, # inconsistent results which depend on the availability of imports + no-self-use, + raise-missing-from, # Python 2.x does not support raise from + too-few-public-methods, + too-many-arguments, + too-many-branches, + too-many-instance-attributes, + too-many-lines, + too-many-locals, + too-many-nested-blocks, + too-many-return-statements, + too-many-statements, + useless-return, # complains about returning None when the return type is optional + # code-smell tests should be updated so the following rules can be enabled + # once that happens the pylint sanity test can be updated to no longer special-case the code-smell tests (use standard ansible-test config instead) + missing-module-docstring, + missing-function-docstring, + +[BASIC] + +bad-names= + _, + bar, + baz, + foo, + tata, + toto, + tutu, + +good-names= + __metaclass__, + C, + ex, + i, + j, + k, + Run, + +class-attribute-rgx=[A-Za-z_][A-Za-z0-9_]{1,40}$ +attr-rgx=[a-z_][a-z0-9_]{1,40}$ +method-rgx=[a-z_][a-z0-9_]{1,40}$ +function-rgx=[a-z_][a-z0-9_]{1,40}$ +module-rgx=[a-z_][a-z0-9_-]{2,40}$ + +[IMPORTS] + +preferred-modules = + distutils.version:ansible.module_utils.compat.version, diff --git a/test/lib/ansible_test/_util/controller/sanity/pylint/plugins/deprecated.py b/test/lib/ansible_test/_util/controller/sanity/pylint/plugins/deprecated.py index e39e5214bf..234ec217cd 100644 --- a/test/lib/ansible_test/_util/controller/sanity/pylint/plugins/deprecated.py +++ b/test/lib/ansible_test/_util/controller/sanity/pylint/plugins/deprecated.py @@ -1,3 +1,4 @@ +"""Ansible specific plyint plugin for checking deprecations.""" # (c) 2018, Matt Martz <matt@sivel.net> # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) # -*- coding: utf-8 -*- @@ -106,6 +107,7 @@ def _get_expr_name(node): def parse_isodate(value): + """Parse an ISO 8601 date string.""" msg = 'Expected ISO 8601 date string (YYYY-MM-DD)' if not isinstance(value, string_types): raise ValueError(msg) @@ -146,10 +148,10 @@ class AnsibleDeprecatedChecker(BaseChecker): def __init__(self, *args, **kwargs): self.collection_version = None self.collection_name = None - super(AnsibleDeprecatedChecker, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def set_option(self, optname, value, action=None, optdict=None): - super(AnsibleDeprecatedChecker, self).set_option(optname, value, action, optdict) + super().set_option(optname, value, action, optdict) if optname == 'collection-version' and value is not None: self.collection_version = SemanticVersion(self.config.collection_version) if optname == 'collection-name' and value is not None: @@ -202,6 +204,7 @@ class AnsibleDeprecatedChecker(BaseChecker): @check_messages(*(MSGS.keys())) def visit_call(self, node): + """Visit a call node.""" version = None date = None collection_name = None diff --git a/test/lib/ansible_test/_util/controller/sanity/pylint/plugins/string_format.py b/test/lib/ansible_test/_util/controller/sanity/pylint/plugins/string_format.py index 1c22a08b97..3b9a37e549 100644 --- a/test/lib/ansible_test/_util/controller/sanity/pylint/plugins/string_format.py +++ b/test/lib/ansible_test/_util/controller/sanity/pylint/plugins/string_format.py @@ -1,3 +1,4 @@ +"""Ansible specific pylint plugin for checking format string usage.""" # (c) 2018, Matt Martz <matt@sivel.net> # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) # -*- coding: utf-8 -*- @@ -41,6 +42,7 @@ class AnsibleStringFormatChecker(BaseChecker): @check_messages(*(MSGS.keys())) def visit_call(self, node): + """Visit a call node.""" func = utils.safe_infer(node.func) if (isinstance(func, astroid.BoundMethod) and isinstance(func.bound, astroid.Instance) diff --git a/test/lib/ansible_test/_util/controller/sanity/validate-modules/main.py b/test/lib/ansible_test/_util/controller/sanity/validate-modules/main.py index c1e2bdaaeb..e6749cdc61 100755..100644 --- a/test/lib/ansible_test/_util/controller/sanity/validate-modules/main.py +++ b/test/lib/ansible_test/_util/controller/sanity/validate-modules/main.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python from __future__ import (absolute_import, division, print_function) __metaclass__ = type diff --git a/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/module_args.py b/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/module_args.py index 8cd0e5e560..1f925bef28 100644 --- a/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/module_args.py +++ b/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/module_args.py @@ -123,7 +123,7 @@ def get_ps_argument_spec(filename, collection): }) script_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'ps_argspec.ps1') - proc = subprocess.Popen([script_path, util_manifest], stdout=subprocess.PIPE, stderr=subprocess.PIPE, + proc = subprocess.Popen(['pwsh', script_path, util_manifest], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False) stdout, stderr = proc.communicate() diff --git a/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/ps_argspec.ps1 b/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/ps_argspec.ps1 index 5ceb9d50b7..fb4a61740a 100755..100644 --- a/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/ps_argspec.ps1 +++ b/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/ps_argspec.ps1 @@ -1,4 +1,3 @@ -#!/usr/bin/env pwsh #Requires -Version 6 Set-StrictMode -Version 2.0 diff --git a/test/lib/ansible_test/_util/controller/sanity/yamllint/yamllinter.py b/test/lib/ansible_test/_util/controller/sanity/yamllint/yamllinter.py index b9fc73e59d..34d2fde99f 100644 --- a/test/lib/ansible_test/_util/controller/sanity/yamllint/yamllinter.py +++ b/test/lib/ansible_test/_util/controller/sanity/yamllint/yamllinter.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """Wrapper around yamllint that supports YAML embedded in Ansible modules.""" from __future__ import (absolute_import, division, print_function) __metaclass__ = type @@ -29,9 +28,9 @@ def main(): class TestConstructor(SafeConstructor): - """Yaml Safe Constructor that knows about Ansible tags""" - + """Yaml Safe Constructor that knows about Ansible tags.""" def construct_yaml_unsafe(self, node): + """Construct an unsafe tag.""" try: constructor = getattr(node, 'id', 'object') if constructor is not None: @@ -60,6 +59,7 @@ TestConstructor.add_constructor( class TestLoader(CParser, TestConstructor, Resolver): + """Custom YAML loader that recognizes custom Ansible tags.""" def __init__(self, stream): CParser.__init__(self, stream) TestConstructor.__init__(self) @@ -92,8 +92,8 @@ class YamlChecker: for path in paths: extension = os.path.splitext(path)[1] - with open(path) as f: - contents = f.read() + with open(path) as file: + contents = file.read() if extension in ('.yml', '.yaml'): self.check_yaml(yaml_conf, path, contents) @@ -150,12 +150,12 @@ class YamlChecker: """ try: yaml.load(contents, Loader=TestLoader) - except MarkedYAMLError as e: + except MarkedYAMLError as ex: self.messages += [{'code': 'unparsable-with-libyaml', - 'message': '%s - %s' % (e.args[0], e.args[2]), + 'message': '%s - %s' % (ex.args[0], ex.args[2]), 'path': path, - 'line': e.problem_mark.line + lineno, - 'column': e.problem_mark.column + 1, + 'line': ex.problem_mark.line + lineno, + 'column': ex.problem_mark.column + 1, 'level': 'error', }] diff --git a/test/lib/ansible_test/_util/controller/tools/sslcheck.py b/test/lib/ansible_test/_util/controller/tools/sslcheck.py index 37b8227936..115c5ed25a 100755..100644 --- a/test/lib/ansible_test/_util/controller/tools/sslcheck.py +++ b/test/lib/ansible_test/_util/controller/tools/sslcheck.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """Show openssl version.""" from __future__ import (absolute_import, division, print_function) __metaclass__ = type diff --git a/test/lib/ansible_test/_util/controller/tools/versions.py b/test/lib/ansible_test/_util/controller/tools/versions.py deleted file mode 100755 index 4babef0162..0000000000 --- a/test/lib/ansible_test/_util/controller/tools/versions.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env python -"""Show python and pip versions.""" -from __future__ import (absolute_import, division, print_function) -__metaclass__ = type - -import os -import sys -import warnings - -warnings.simplefilter('ignore') # avoid python version deprecation warnings when using newer pip dependencies - -try: - import pip -except ImportError: - pip = None - -print(sys.version) - -if pip: - print('pip %s from %s' % (pip.__version__, os.path.dirname(pip.__file__))) diff --git a/test/lib/ansible_test/_util/controller/tools/virtualenvcheck.py b/test/lib/ansible_test/_util/controller/tools/virtualenvcheck.py index 0c8f768034..90dfa39410 100755..100644 --- a/test/lib/ansible_test/_util/controller/tools/virtualenvcheck.py +++ b/test/lib/ansible_test/_util/controller/tools/virtualenvcheck.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """Detect the real python interpreter when running in a virtual environment created by the 'virtualenv' module.""" from __future__ import (absolute_import, division, print_function) __metaclass__ = type diff --git a/test/lib/ansible_test/_util/controller/tools/yamlcheck.py b/test/lib/ansible_test/_util/controller/tools/yamlcheck.py index 591842f4ad..dfd08e581c 100755..100644 --- a/test/lib/ansible_test/_util/controller/tools/yamlcheck.py +++ b/test/lib/ansible_test/_util/controller/tools/yamlcheck.py @@ -1,5 +1,4 @@ -#!/usr/bin/env python -"""Show python and pip versions.""" +"""Show availability of PyYAML and libyaml support.""" from __future__ import (absolute_import, division, print_function) __metaclass__ = type diff --git a/test/lib/ansible_test/_util/target/__init__.py b/test/lib/ansible_test/_util/target/__init__.py new file mode 100644 index 0000000000..d6fc0a8614 --- /dev/null +++ b/test/lib/ansible_test/_util/target/__init__.py @@ -0,0 +1,3 @@ +"""Nearly empty __init__.py to allow importing under Python 2.x.""" +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type diff --git a/test/lib/ansible_test/_util/target/cli/ansible_test_cli_stub.py b/test/lib/ansible_test/_util/target/cli/ansible_test_cli_stub.py new file mode 100755 index 0000000000..dc31095a81 --- /dev/null +++ b/test/lib/ansible_test/_util/target/cli/ansible_test_cli_stub.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python +# PYTHON_ARGCOMPLETE_OK +"""Command line entry point for ansible-test.""" + +# NOTE: This file resides in the _util/target directory to ensure compatibility with all supported Python versions. + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import os +import sys + + +def main(): + """Main program entry point.""" + ansible_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + source_root = os.path.join(ansible_root, 'test', 'lib') + + if os.path.exists(os.path.join(source_root, 'ansible_test', '_internal', '__init__.py')): + # running from source, use that version of ansible-test instead of any version that may already be installed + sys.path.insert(0, source_root) + + # noinspection PyProtectedMember + from ansible_test._util.target.common.constants import CONTROLLER_PYTHON_VERSIONS + + if version_to_str(sys.version_info[:2]) not in CONTROLLER_PYTHON_VERSIONS: + raise SystemExit('This version of ansible-test cannot be executed with Python version %s. Supported Python versions are: %s' % ( + version_to_str(sys.version_info[:3]), ', '.join(CONTROLLER_PYTHON_VERSIONS))) + + # noinspection PyProtectedMember + from ansible_test._internal import main as cli_main + + cli_main() + + +def version_to_str(version): + """Return a version string from a version tuple.""" + return '.'.join(str(n) for n in version) + + +if __name__ == '__main__': + main() diff --git a/test/lib/ansible_test/_util/target/common/__init__.py b/test/lib/ansible_test/_util/target/common/__init__.py new file mode 100644 index 0000000000..d6fc0a8614 --- /dev/null +++ b/test/lib/ansible_test/_util/target/common/__init__.py @@ -0,0 +1,3 @@ +"""Nearly empty __init__.py to allow importing under Python 2.x.""" +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type diff --git a/test/lib/ansible_test/_util/target/common/constants.py b/test/lib/ansible_test/_util/target/common/constants.py new file mode 100644 index 0000000000..9902b046a6 --- /dev/null +++ b/test/lib/ansible_test/_util/target/common/constants.py @@ -0,0 +1,45 @@ +"""Constants used by ansible-test. Imports should not be used in this file.""" + +# NOTE: This file resides in the _util/target directory to ensure compatibility with all supported Python versions. + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +# Setting a low soft RLIMIT_NOFILE value will improve the performance of subprocess.Popen on Python 2.x when close_fds=True. +# This will affect all Python subprocesses. It will also affect the current Python process if set before subprocess is imported for the first time. +SOFT_RLIMIT_NOFILE = 1024 + +# File used to track the ansible-test test execution timeout. +TIMEOUT_PATH = '.ansible-test-timeout.json' + +REMOTE_ONLY_PYTHON_VERSIONS = ( + '2.6', + '2.7', + '3.5', + '3.6', + '3.7', +) + +CONTROLLER_PYTHON_VERSIONS = ( + '3.8', + '3.9', + '3.10', +) + +CONTROLLER_MIN_PYTHON_VERSION = CONTROLLER_PYTHON_VERSIONS[0] +SUPPORTED_PYTHON_VERSIONS = REMOTE_ONLY_PYTHON_VERSIONS + CONTROLLER_PYTHON_VERSIONS + +COVERAGE_REQUIRED_VERSION = '4.5.4' + +REMOTE_PROVIDERS = [ + 'default', + 'aws', + 'azure', + 'ibmps', + 'parallels', +] + +SECCOMP_CHOICES = [ + 'default', + 'unconfined', +] diff --git a/test/lib/ansible_test/_util/target/sanity/compile/compile.py b/test/lib/ansible_test/_util/target/sanity/compile/compile.py index 3f6fc96260..e2302fc0e7 100755..100644 --- a/test/lib/ansible_test/_util/target/sanity/compile/compile.py +++ b/test/lib/ansible_test/_util/target/sanity/compile/compile.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """Python syntax checker with lint friendly output.""" from __future__ import (absolute_import, division, print_function) __metaclass__ = type @@ -11,6 +10,7 @@ Text = type(u'') def main(): + """Main program entry point.""" for path in sys.argv[1:] or sys.stdin.read().splitlines(): with open(path, 'rb') as source_fd: source = source_fd.read() diff --git a/test/lib/ansible_test/_util/target/sanity/import/yaml_to_json.py b/test/lib/ansible_test/_util/target/sanity/import/yaml_to_json.py index 09be9576d9..1164168e3e 100644 --- a/test/lib/ansible_test/_util/target/sanity/import/yaml_to_json.py +++ b/test/lib/ansible_test/_util/target/sanity/import/yaml_to_json.py @@ -18,6 +18,7 @@ ISO_DATE_MARKER = 'isodate:f23983df-f3df-453c-9904-bcd08af468cc:' def default(value): + """Custom default serializer which supports datetime.date types.""" if isinstance(value, datetime.date): return '%s%s' % (ISO_DATE_MARKER, value.isoformat()) diff --git a/test/lib/ansible_test/_util/target/setup/bootstrap.sh b/test/lib/ansible_test/_util/target/setup/bootstrap.sh new file mode 100644 index 0000000000..36ca68f494 --- /dev/null +++ b/test/lib/ansible_test/_util/target/setup/bootstrap.sh @@ -0,0 +1,323 @@ +#!/bin/sh + +set -eu + +install_ssh_keys() +{ + if [ ! -f "${ssh_private_key_path}" ]; then + # write public/private ssh key pair + public_key_path="${ssh_private_key_path}.pub" + + # shellcheck disable=SC2174 + mkdir -m 0700 -p "${ssh_path}" + touch "${public_key_path}" "${ssh_private_key_path}" + chmod 0600 "${public_key_path}" "${ssh_private_key_path}" + echo "${ssh_public_key}" > "${public_key_path}" + echo "${ssh_private_key}" > "${ssh_private_key_path}" + + # add public key to authorized_keys + authoried_keys_path="${HOME}/.ssh/authorized_keys" + + # the existing file is overwritten to avoid conflicts (ex: RHEL on EC2 blocks root login) + cat "${public_key_path}" > "${authoried_keys_path}" + chmod 0600 "${authoried_keys_path}" + + # add localhost's server keys to known_hosts + known_hosts_path="${HOME}/.ssh/known_hosts" + + for key in /etc/ssh/ssh_host_*_key.pub; do + echo "localhost $(cat "${key}")" >> "${known_hosts_path}" + done + fi +} + +customize_bashrc() +{ + true > ~/.bashrc + + # Show color `ls` results when available. + if ls --color > /dev/null 2>&1; then + echo "alias ls='ls --color'" >> ~/.bashrc + elif ls -G > /dev/null 2>&1; then + echo "alias ls='ls -G'" >> ~/.bashrc + fi + + # Improve shell prompts for interactive use. + echo "export PS1='\[\e]0;\u@\h: \w\a\]\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ '" >> ~/.bashrc +} + +install_pip() { + if ! "${python_interpreter}" -m pip.__main__ --version --disable-pip-version-check 2>/dev/null; then + case "${python_version}" in + *) + pip_bootstrap_url="https://ansible-ci-files.s3.amazonaws.com/ansible-test/get-pip-20.3.4.py" + ;; + esac + + while true; do + curl --silent --show-error "${pip_bootstrap_url}" -o /tmp/get-pip.py && \ + "${python_interpreter}" /tmp/get-pip.py --disable-pip-version-check --quiet && \ + rm /tmp/get-pip.py \ + && break + echo "Failed to install packages. Sleeping before trying again..." + sleep 10 + done + fi +} + +pip_install() { + pip_packages="$1" + + while true; do + # shellcheck disable=SC2086 + "${python_interpreter}" -m pip install --disable-pip-version-check ${pip_packages} \ + && break + echo "Failed to install packages. Sleeping before trying again..." + sleep 10 + done +} + +bootstrap_remote_aix() +{ + chfs -a size=1G / + chfs -a size=4G /usr + chfs -a size=1G /var + chfs -a size=1G /tmp + chfs -a size=2G /opt + + if [ "${python_version}" = "2.7" ]; then + python_package_version="" + else + python_package_version="3" + fi + + packages=" + gcc + python${python_package_version} + python${python_package_version}-devel + python${python_package_version}-pip + " + + while true; do + # shellcheck disable=SC2086 + yum install -q -y ${packages} \ + && break + echo "Failed to install packages. Sleeping before trying again..." + sleep 10 + done +} + +bootstrap_remote_freebsd() +{ + if [ "${python_version}" = "2.7" ]; then + # on Python 2.7 our only option is to use virtualenv + virtualenv_pkg="py27-virtualenv" + else + # on Python 3.x we'll use the built-in venv instead + virtualenv_pkg="" + fi + + packages=" + python${python_package_version} + ${virtualenv_pkg} + bash + curl + gtar + sudo + " + + if [ "${controller}" ]; then + # Declare platform/python version combinations which do not have supporting OS packages available. + # For these combinations ansible-test will use pip to install the requirements instead. + case "${platform_version}/${python_version}" in + "11.4/3.8") + have_os_packages="" + ;; + "12.2/3.8") + have_os_packages="" + ;; + "13.0/3.8") + have_os_packages="" + ;; + "13.0/3.9") + have_os_packages="" + ;; + *) + have_os_packages="yes" + ;; + esac + + # PyYAML is never installed with an OS package since it does not include libyaml support. + # Instead, ansible-test will install it using pip. + if [ "${have_os_packages}" ]; then + jinja2_pkg="py${python_package_version}-Jinja2" + cryptography_pkg="py${python_package_version}-cryptography" + else + jinja2_pkg="" + cryptography_pkg="" + fi + + packages=" + ${packages} + libyaml + ${jinja2_pkg} + ${cryptography_pkg} + " + fi + + while true; do + # shellcheck disable=SC2086 + env ASSUME_ALWAYS_YES=YES pkg bootstrap && \ + pkg install -q -y ${packages} \ + && break + echo "Failed to install packages. Sleeping before trying again..." + sleep 10 + done + + install_pip + + if ! grep '^PermitRootLogin yes$' /etc/ssh/sshd_config > /dev/null; then + sed -i '' 's/^# *PermitRootLogin.*$/PermitRootLogin yes/;' /etc/ssh/sshd_config + service sshd restart + fi +} + +bootstrap_remote_macos() +{ + # Silence macOS deprecation warning for bash. + echo "export BASH_SILENCE_DEPRECATION_WARNING=1" >> ~/.bashrc + + # Make sure ~/ansible/ is the starting directory for interactive shells on the control node. + # The root home directory is under a symlink. Without this the real path will be displayed instead. + if [ "${controller}" ]; then + echo "cd ~/ansible/" >> ~/.bashrc + fi + + # Make sure commands like 'brew' can be found. + # This affects users with the 'zsh' shell, as well as 'root' accessed using 'sudo' from a user with 'zsh' for a shell. + # shellcheck disable=SC2016 + echo 'PATH="/usr/local/bin:$PATH"' > /etc/zshenv +} + +bootstrap_remote_rhel_7() +{ + packages=" + gcc + python-devel + python-virtualenv + " + + if [ "${controller}" ]; then + packages=" + ${packages} + python2-cryptography + " + fi + + while true; do + # shellcheck disable=SC2086 + yum install -q -y ${packages} \ + && break + echo "Failed to install packages. Sleeping before trying again..." + sleep 10 + done + + install_pip +} + +bootstrap_remote_rhel_8() +{ + if [ "${python_version}" = "3.6" ]; then + py_pkg_prefix="python3" + else + py_pkg_prefix="python${python_package_version}" + fi + + packages=" + gcc + ${py_pkg_prefix}-devel + " + + if [ "${controller}" ]; then + packages=" + ${packages} + ${py_pkg_prefix}-jinja2 + ${py_pkg_prefix}-cryptography + " + fi + + while true; do + # shellcheck disable=SC2086 + yum module install -q -y "python${python_package_version}" && \ + yum install -q -y ${packages} \ + && break + echo "Failed to install packages. Sleeping before trying again..." + sleep 10 + done +} + +bootstrap_remote_rhel() +{ + case "${platform_version}" in + 7.*) bootstrap_remote_rhel_7 ;; + 8.*) bootstrap_remote_rhel_8 ;; + esac + + # pin packaging and pyparsing to match the downstream vendored versions + pip_packages=" + packaging==20.4 + pyparsing==2.4.7 + " + + pip_install "${pip_packages}" +} + +bootstrap_docker() +{ + # Required for newer mysql-server packages to install/upgrade on Ubuntu 16.04. + rm -f /usr/sbin/policy-rc.d +} + +bootstrap_remote() +{ + for python_version in ${python_versions}; do + echo "Bootstrapping Python ${python_version}" + + python_interpreter="python${python_version}" + python_package_version="$(echo "${python_version}" | tr -d '.')" + + case "${platform}" in + "aix") bootstrap_remote_aix ;; + "freebsd") bootstrap_remote_freebsd ;; + "macos") bootstrap_remote_macos ;; + "rhel") bootstrap_remote_rhel ;; + esac + done +} + +bootstrap() +{ + ssh_path="${HOME}/.ssh" + ssh_private_key_path="${ssh_path}/id_${ssh_key_type}" + + install_ssh_keys + customize_bashrc + + case "${bootstrap_type}" in + "docker") bootstrap_docker ;; + "remote") bootstrap_remote ;; + esac +} + +# These variables will be templated before sending the script to the host. +# They are at the end of the script to maintain line numbers for debugging purposes. +bootstrap_type=#{bootstrap_type} +controller=#{controller} +platform=#{platform} +platform_version=#{platform_version} +python_versions=#{python_versions} +ssh_key_type=#{ssh_key_type} +ssh_private_key=#{ssh_private_key} +ssh_public_key=#{ssh_public_key} + +bootstrap diff --git a/test/lib/ansible_test/_util/target/setup/docker.sh b/test/lib/ansible_test/_util/target/setup/docker.sh deleted file mode 100644 index ea60e1a6f3..0000000000 --- a/test/lib/ansible_test/_util/target/setup/docker.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh - -set -eu - -# Required for newer mysql-server packages to install/upgrade on Ubuntu 16.04. -rm -f /usr/sbin/policy-rc.d - -# Improve prompts on remote host for interactive use. -# `cat << EOF > ~/.bashrc` flakes sometimes since /tmp may not be ready yet in -# the container. So don't do that -echo "alias ls='ls --color=auto'" > ~/.bashrc -echo "export PS1='\[\e]0;\u@\h: \w\a\]\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ '" >> ~/.bashrc -echo "cd ~/ansible/" >> ~/.bashrc diff --git a/test/lib/ansible_test/_util/controller/tools/quiet_pip.py b/test/lib/ansible_test/_util/target/setup/quiet_pip.py index e1bb824646..83d4576b08 100644 --- a/test/lib/ansible_test/_util/controller/tools/quiet_pip.py +++ b/test/lib/ansible_test/_util/target/setup/quiet_pip.py @@ -11,6 +11,7 @@ BUILTIN_FILTERER_FILTER = logging.Filterer.filter LOGGING_MESSAGE_FILTER = re.compile("^(" ".*Running pip install with root privileges is generally not a good idea.*|" # custom Fedora patch [1] + ".*Running pip as the 'root' user can result in broken permissions .*|" # pip 21.1 "DEPRECATION: Python 2.7 will reach the end of its life .*|" # pip 19.2.3 "Ignoring .*: markers .* don't match your environment|" "Looking in indexes: .*|" # pypi-test-container diff --git a/test/lib/ansible_test/_util/target/setup/remote.sh b/test/lib/ansible_test/_util/target/setup/remote.sh deleted file mode 100644 index 9348ac6f9f..0000000000 --- a/test/lib/ansible_test/_util/target/setup/remote.sh +++ /dev/null @@ -1,185 +0,0 @@ -#!/bin/sh - -set -eu - -platform=#{platform} -platform_version=#{platform_version} -python_version=#{python_version} - -python_interpreter="python${python_version}" - -cd ~/ - -install_pip () { - if ! "${python_interpreter}" -m pip.__main__ --version --disable-pip-version-check 2>/dev/null; then - case "${python_version}" in - *) - pip_bootstrap_url="https://ansible-ci-files.s3.amazonaws.com/ansible-test/get-pip-20.3.4.py" - ;; - esac - curl --silent --show-error "${pip_bootstrap_url}" -o /tmp/get-pip.py - "${python_interpreter}" /tmp/get-pip.py --disable-pip-version-check --quiet - rm /tmp/get-pip.py - fi -} - -if [ "${platform}" = "freebsd" ]; then - py_version="$(echo "${python_version}" | tr -d '.')" - - if [ "${py_version}" = "27" ]; then - # on Python 2.7 our only option is to use virtualenv - virtualenv_pkg="py27-virtualenv" - else - # on Python 3.x we'll use the built-in venv instead - virtualenv_pkg="" - fi - - # Declare platform/python version combinations which do not have supporting OS packages available. - # For these combinations ansible-test will use pip to install the requirements instead. - case "${platform_version}/${python_version}" in - "11.4/3.8") - have_os_packages="" - ;; - "12.2/3.8") - have_os_packages="" - ;; - *) - have_os_packages="yes" - ;; - esac - - # PyYAML is never installed with an OS package since it does not include libyaml support. - # Instead, ansible-test will always install it using pip. - if [ "${have_os_packages}" ]; then - jinja2_pkg="py${py_version}-Jinja2" - cryptography_pkg="py${py_version}-cryptography" - else - jinja2_pkg="" - cryptography_pkg="" - fi - - while true; do - # shellcheck disable=SC2086 - env ASSUME_ALWAYS_YES=YES pkg bootstrap && \ - pkg install -q -y \ - bash \ - curl \ - gtar \ - libyaml \ - "python${py_version}" \ - ${jinja2_pkg} \ - ${cryptography_pkg} \ - ${virtualenv_pkg} \ - sudo \ - && break - echo "Failed to install packages. Sleeping before trying again..." - sleep 10 - done - - install_pip - - if ! grep '^PermitRootLogin yes$' /etc/ssh/sshd_config > /dev/null; then - sed -i '' 's/^# *PermitRootLogin.*$/PermitRootLogin yes/;' /etc/ssh/sshd_config - service sshd restart - fi -elif [ "${platform}" = "rhel" ]; then - if grep '8\.' /etc/redhat-release; then - py_version="$(echo "${python_version}" | tr -d '.')" - - if [ "${py_version}" = "36" ]; then - py_pkg_prefix="python3" - else - py_pkg_prefix="python${py_version}" - fi - - while true; do - yum module install -q -y "python${py_version}" && \ - yum install -q -y \ - gcc \ - "${py_pkg_prefix}-devel" \ - "${py_pkg_prefix}-jinja2" \ - "${py_pkg_prefix}-cryptography" \ - iptables \ - && break - echo "Failed to install packages. Sleeping before trying again..." - sleep 10 - done - else - while true; do - yum install -q -y \ - gcc \ - python-devel \ - python-virtualenv \ - python2-cryptography \ - && break - echo "Failed to install packages. Sleeping before trying again..." - sleep 10 - done - - install_pip - fi - - # pin packaging and pyparsing to match the downstream vendored versions - "${python_interpreter}" -m pip install packaging==20.4 pyparsing==2.4.7 --disable-pip-version-check -elif [ "${platform}" = "centos" ]; then - while true; do - yum install -q -y \ - gcc \ - python-devel \ - python-virtualenv \ - python2-cryptography \ - libffi-devel \ - openssl-devel \ - && break - echo "Failed to install packages. Sleeping before trying again..." - sleep 10 - done - - install_pip -elif [ "${platform}" = "osx" ]; then - while true; do - pip install --disable-pip-version-check --quiet \ - 'virtualenv==16.7.10' \ - && break - echo "Failed to install packages. Sleeping before trying again..." - sleep 10 - done -elif [ "${platform}" = "aix" ]; then - chfs -a size=1G / - chfs -a size=4G /usr - chfs -a size=1G /var - chfs -a size=1G /tmp - chfs -a size=2G /opt - while true; do - yum install -q -y \ - gcc \ - libffi-devel \ - python-jinja2 \ - python-cryptography \ - python-pip && \ - pip install --disable-pip-version-check --quiet \ - 'virtualenv==16.7.10' \ - && break - echo "Failed to install packages. Sleeping before trying again..." - sleep 10 - done -fi - -# Improve prompts on remote host for interactive use. -# shellcheck disable=SC1117 -cat << EOF > ~/.bashrc -if ls --color > /dev/null 2>&1; then - alias ls='ls --color' -elif ls -G > /dev/null 2>&1; then - alias ls='ls -G' -fi -export PS1='\[\e]0;\u@\h: \w\a\]\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ ' -EOF - -# Make sure ~/ansible/ is the starting directory for interactive shells. -if [ "${platform}" = "osx" ]; then - echo "cd ~/ansible/" >> ~/.bashrc -elif [ "${platform}" = "macos" ] ; then - echo "export BASH_SILENCE_DEPRECATION_WARNING=1" >> ~/.bashrc - echo "cd ~/ansible/" >> ~/.bashrc -fi diff --git a/test/lib/ansible_test/_util/target/setup/requirements.py b/test/lib/ansible_test/_util/target/setup/requirements.py new file mode 100644 index 0000000000..0e3b1e634a --- /dev/null +++ b/test/lib/ansible_test/_util/target/setup/requirements.py @@ -0,0 +1,252 @@ +"""A tool for installing test requirements on the controller and target host.""" +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +# pylint: disable=wrong-import-position + +import resource + +# Setting a low soft RLIMIT_NOFILE value will improve the performance of subprocess.Popen on Python 2.x when close_fds=True. +# This will affect all Python subprocesses. It will also affect the current Python process if set before subprocess is imported for the first time. +SOFT_RLIMIT_NOFILE = 1024 + +CURRENT_RLIMIT_NOFILE = resource.getrlimit(resource.RLIMIT_NOFILE) +DESIRED_RLIMIT_NOFILE = (SOFT_RLIMIT_NOFILE, CURRENT_RLIMIT_NOFILE[1]) + +if DESIRED_RLIMIT_NOFILE < CURRENT_RLIMIT_NOFILE: + resource.setrlimit(resource.RLIMIT_NOFILE, DESIRED_RLIMIT_NOFILE) + CURRENT_RLIMIT_NOFILE = DESIRED_RLIMIT_NOFILE + +import base64 +import errno +import io +import json +import os +import shutil +import subprocess +import sys +import tempfile + +try: + import typing as t +except ImportError: + t = None + +try: + from shlex import quote as cmd_quote +except ImportError: + # noinspection PyProtectedMember + from pipes import quote as cmd_quote + +ENCODING = 'utf-8' +PAYLOAD = b'{payload}' # base-64 encoded JSON payload which will be populated before this script is executed + +Text = type(u'') + +VERBOSITY = 0 +CONSOLE = sys.stderr + + +def main(): # type: () -> None + """Main program entry point.""" + global VERBOSITY # pylint: disable=global-statement + + payload = json.loads(to_text(base64.b64decode(PAYLOAD))) + + VERBOSITY = payload['verbosity'] + + script = payload['script'] + commands = payload['commands'] + + with tempfile.NamedTemporaryFile(prefix='ansible-test-', suffix='-pip.py') as pip: + pip.write(to_bytes(script)) + pip.flush() + + for name, options in commands: + try: + globals()[name](pip.name, options) + except ApplicationError as ex: + print(ex) + sys.exit(1) + + +def install(pip, options): # type: (str, t.Dict[str, t.Any]) -> None + """Perform a pip install.""" + requirements = options['requirements'] + constraints = options['constraints'] + packages = options['packages'] + + tempdir = tempfile.mkdtemp(prefix='ansible-test-', suffix='-requirements') + + try: + options = common_pip_options() + options.extend(packages) + + for path, content in requirements: + write_text_file(os.path.join(tempdir, path), content, True) + options.extend(['-r', path]) + + for path, content in constraints: + write_text_file(os.path.join(tempdir, path), content, True) + options.extend(['-c', path]) + + command = [sys.executable, pip, 'install'] + options + + execute_command(command, tempdir) + finally: + remove_tree(tempdir) + + +def uninstall(pip, options): # type: (str, t.Dict[str, t.Any]) -> None + """Perform a pip uninstall.""" + packages = options['packages'] + ignore_errors = options['ignore_errors'] + + options = common_pip_options() + options.extend(packages) + + command = [sys.executable, pip, 'uninstall', '-y'] + options + + try: + execute_command(command, capture=True) + except SubprocessError: + if not ignore_errors: + raise + + +def common_pip_options(): # type: () -> t.List[str] + """Return a list of common pip options.""" + return [ + '--disable-pip-version-check', + ] + + +def devnull(): # type: () -> t.IO[bytes] + """Return a file object that references devnull.""" + try: + return devnull.file + except AttributeError: + devnull.file = open(os.devnull, 'w+b') # pylint: disable=consider-using-with + + return devnull.file + + +class ApplicationError(Exception): + """Base class for application exceptions.""" + + +class SubprocessError(ApplicationError): + """A command returned a non-zero status.""" + def __init__(self, cmd, status, stdout, stderr): # type: (t.List[str], int, str, str) -> None + message = 'A command failed with status %d: %s' % (status, ' '.join(cmd_quote(c) for c in cmd)) + + if stderr: + message += '\n>>> Standard Error\n%s' % stderr.strip() + + if stdout: + message += '\n>>> Standard Output\n%s' % stdout.strip() + + super(SubprocessError, self).__init__(message) + + +def log(message, verbosity=0): # type: (str, int) -> None + """Log a message to the console if the verbosity is high enough.""" + if verbosity > VERBOSITY: + return + + print(message, file=CONSOLE) + CONSOLE.flush() + + +def execute_command(cmd, cwd=None, capture=False): # type: (t.List[str], t.Optional[str], bool) -> None + """Execute the specified command.""" + log('Execute command: %s' % ' '.join(cmd_quote(c) for c in cmd), verbosity=1) + + cmd_bytes = [to_bytes(c) for c in cmd] + + if capture: + stdout = subprocess.PIPE + stderr = subprocess.PIPE + else: + stdout = None + stderr = None + + process = subprocess.Popen(cmd_bytes, cwd=to_optional_bytes(cwd), stdin=devnull(), stdout=stdout, stderr=stderr) # pylint: disable=consider-using-with + stdout_bytes, stderr_bytes = process.communicate() + stdout_text = to_optional_text(stdout_bytes) or u'' + stderr_text = to_optional_text(stderr_bytes) or u'' + + if process.returncode != 0: + raise SubprocessError(cmd, process.returncode, stdout_text, stderr_text) + + +def write_text_file(path, content, create_directories=False): # type: (str, str, bool) -> None + """Write the given text content to the specified path, optionally creating missing directories.""" + if create_directories: + make_dirs(os.path.dirname(path)) + + with open_binary_file(path, 'wb') as file_obj: + file_obj.write(to_bytes(content)) + + +def remove_tree(path): # type: (str) -> None + """Remove the specified directory tree.""" + try: + shutil.rmtree(to_bytes(path)) + except OSError as ex: + if ex.errno != errno.ENOENT: + raise + + +def make_dirs(path): # type: (str) -> None + """Create a directory at path, including any necessary parent directories.""" + try: + os.makedirs(to_bytes(path)) + except OSError as ex: + if ex.errno != errno.EEXIST: + raise + + +def open_binary_file(path, mode='rb'): # type: (str, str) -> t.BinaryIO + """Open the given path for binary access.""" + if 'b' not in mode: + raise Exception('mode must include "b" for binary files: %s' % mode) + + # noinspection PyTypeChecker + return io.open(to_bytes(path), mode) # pylint: disable=consider-using-with + + +def to_optional_bytes(value, errors='strict'): # type: (t.Optional[t.AnyStr], str) -> t.Optional[bytes] + """Return the given value as bytes encoded using UTF-8 if not already bytes, or None if the value is None.""" + return None if value is None else to_bytes(value, errors) + + +def to_optional_text(value, errors='strict'): # type: (t.Optional[t.AnyStr], str) -> t.Optional[t.Text] + """Return the given value as text decoded using UTF-8 if not already text, or None if the value is None.""" + return None if value is None else to_text(value, errors) + + +def to_bytes(value, errors='strict'): # type: (t.AnyStr, str) -> bytes + """Return the given value as bytes encoded using UTF-8 if not already bytes.""" + if isinstance(value, bytes): + return value + + if isinstance(value, Text): + return value.encode(ENCODING, errors) + + raise Exception('value is not bytes or text: %s' % type(value)) + + +def to_text(value, errors='strict'): # type: (t.AnyStr, str) -> t.Text + """Return the given value as text decoded using UTF-8 if not already text.""" + if isinstance(value, bytes): + return value.decode(ENCODING, errors) + + if isinstance(value, Text): + return value + + raise Exception('value is not bytes or text: %s' % type(value)) + + +if __name__ == '__main__': + main() diff --git a/test/lib/ansible_test/_util/target/setup/ssh-keys.sh b/test/lib/ansible_test/_util/target/setup/ssh-keys.sh deleted file mode 100644 index 7846f3fef0..0000000000 --- a/test/lib/ansible_test/_util/target/setup/ssh-keys.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/sh -# Configure SSH keys. - -ssh_public_key=#{ssh_public_key} -ssh_private_key=#{ssh_private_key} -ssh_key_type=#{ssh_key_type} - -ssh_path="${HOME}/.ssh" -private_key_path="${ssh_path}/id_${ssh_key_type}" - -if [ ! -f "${private_key_path}" ]; then - # write public/private ssh key pair - public_key_path="${private_key_path}.pub" - - # shellcheck disable=SC2174 - mkdir -m 0700 -p "${ssh_path}" - touch "${public_key_path}" "${private_key_path}" - chmod 0600 "${public_key_path}" "${private_key_path}" - echo "${ssh_public_key}" > "${public_key_path}" - echo "${ssh_private_key}" > "${private_key_path}" - - # add public key to authorized_keys - authoried_keys_path="${HOME}/.ssh/authorized_keys" - - # the existing file is overwritten to avoid conflicts (ex: RHEL on EC2 blocks root login) - cat "${public_key_path}" > "${authoried_keys_path}" - chmod 0600 "${authoried_keys_path}" - - # add localhost's server keys to known_hosts - known_hosts_path="${HOME}/.ssh/known_hosts" - - for key in /etc/ssh/ssh_host_*_key.pub; do - echo "localhost $(cat "${key}")" >> "${known_hosts_path}" - done -fi |