diff options
89 files changed, 747 insertions, 882 deletions
diff --git a/bin/ansible b/bin/ansible index be4edc2308..c7abff0180 100755 --- a/bin/ansible +++ b/bin/ansible @@ -43,7 +43,7 @@ from multiprocessing import Lock import ansible.constants as C from ansible.errors import AnsibleError, AnsibleOptionsError, AnsibleParserError from ansible.utils.display import Display -from ansible.utils.unicode import to_unicode +from ansible.module_utils._text import to_text ######################################## @@ -97,10 +97,10 @@ if __name__ == '__main__': except AnsibleOptionsError as e: cli.parser.print_help() - display.error(to_unicode(e), wrap_text=False) + display.error(to_text(e), wrap_text=False) exit_code = 5 except AnsibleParserError as e: - display.error(to_unicode(e), wrap_text=False) + display.error(to_text(e), wrap_text=False) exit_code = 4 # TQM takes care of these, but leaving comment to reserve the exit codes # except AnsibleHostUnreachable as e: @@ -110,16 +110,16 @@ if __name__ == '__main__': # display.error(str(e)) # exit_code = 2 except AnsibleError as e: - display.error(to_unicode(e), wrap_text=False) + display.error(to_text(e), wrap_text=False) exit_code = 1 except KeyboardInterrupt: display.error("User interrupted execution") exit_code = 99 except Exception as e: have_cli_options = cli is not None and cli.options is not None - display.error("Unexpected Exception: %s" % to_unicode(e), wrap_text=False) + display.error("Unexpected Exception: %s" % to_text(e), wrap_text=False) if not have_cli_options or have_cli_options and cli.options.verbosity > 2: - display.display(u"the full traceback was:\n\n%s" % to_unicode(traceback.format_exc())) + display.display(u"the full traceback was:\n\n%s" % to_text(traceback.format_exc())) else: display.display("to see the full traceback, use -vvv") exit_code = 250 diff --git a/docsite/rst/developing_modules_python3.rst b/docsite/rst/developing_modules_python3.rst index 26fdc9ad7f..901dfaed17 100644 --- a/docsite/rst/developing_modules_python3.rst +++ b/docsite/rst/developing_modules_python3.rst @@ -215,6 +215,28 @@ Python3. We'll need to gather experience to see if this is going to work out well for modules as well or if we should give the module_utils API explicit switches so that modules can choose to operate with text type all of the time. +Helpers +~~~~~~~ + +For converting between bytes, text, and native strings we have three helper +functions. These are :func:`ansible.module_utils._text.to_bytes`, +:func:`ansible.module_utils._text.to_native`, and +:func:`ansible.module_utils._text.to_text`. These are similar to using +``bytes.decode()`` and ``unicode.encode()`` with a few differences. + +* By default they try very hard not to traceback. +* The default encoding is "utf-8" +* There are two error strategies that don't correspond one-to-one with + a python codec error handler. These are ``surrogate_or_strict`` and + ``surrogate_or_replace``. ``surrogate_or_strict`` will use the ``surrogateescape`` + error handler if available (mostly on python3) or strict if not. It is most + appropriate to use when dealing with something that needs to round trip its + value like file paths database keys, etc. Without ``surrogateescape`` the best + thing these values can do is generate a traceback that our code can catch + and decide how to show an error message. ``surrogate_or_replace`` is for + when a value is going to be displayed to the user. If the + ``surrogateescape`` error handler is not present, it will replace + undecodable byte sequences with a replacement character. ================================ Porting Core Ansible to Python 3 diff --git a/hacking/module_formatter.py b/hacking/module_formatter.py index 7a4005c044..6e84cada28 100755 --- a/hacking/module_formatter.py +++ b/hacking/module_formatter.py @@ -19,6 +19,7 @@ # from __future__ import print_function +__metaclass__ = type import os import glob @@ -34,10 +35,10 @@ from collections import defaultdict from jinja2 import Environment, FileSystemLoader from six import iteritems +from ansible.errors import AnsibleError +from ansible.module_utils._text import to_bytes from ansible.utils import module_docs from ansible.utils.vars import merge_hash -from ansible.utils.unicode import to_bytes -from ansible.errors import AnsibleError ##################################################################################### # constants and paths diff --git a/lib/ansible/cli/__init__.py b/lib/ansible/cli/__init__.py index b12fc0dbc6..3176d9c956 100644 --- a/lib/ansible/cli/__init__.py +++ b/lib/ansible/cli/__init__.py @@ -33,7 +33,7 @@ import subprocess from ansible.release import __version__ from ansible import constants as C from ansible.errors import AnsibleError, AnsibleOptionsError -from ansible.utils.unicode import to_bytes, to_unicode +from ansible.module_utils._text import to_bytes, to_text try: from __main__ import display @@ -109,7 +109,7 @@ class CLI(object): if self.options.verbosity > 0: if C.CONFIG_FILE: - display.display(u"Using %s as config file" % to_unicode(C.CONFIG_FILE)) + display.display(u"Using %s as config file" % to_text(C.CONFIG_FILE)) else: display.display(u"No config file found; using defaults") diff --git a/lib/ansible/cli/adhoc.py b/lib/ansible/cli/adhoc.py index cfaeca955b..95ab640ded 100644 --- a/lib/ansible/cli/adhoc.py +++ b/lib/ansible/cli/adhoc.py @@ -27,13 +27,13 @@ from ansible.cli import CLI from ansible.errors import AnsibleError, AnsibleOptionsError from ansible.executor.task_queue_manager import TaskQueueManager from ansible.inventory import Inventory +from ansible.module_utils._text import to_text from ansible.parsing.dataloader import DataLoader from ansible.parsing.splitter import parse_kv from ansible.playbook.play import Play from ansible.plugins import get_all_plugin_loaders from ansible.utils.vars import load_extra_vars from ansible.utils.vars import load_options_vars -from ansible.utils.unicode import to_unicode from ansible.vars import VariableManager try: @@ -99,7 +99,7 @@ class AdHocCLI(CLI): super(AdHocCLI, self).run() # only thing left should be host pattern - pattern = to_unicode(self.args[0], errors='strict') + pattern = to_text(self.args[0], errors='surrogate_or_strict') # ignore connection password cause we are local if self.options.connection == "local": @@ -169,7 +169,7 @@ class AdHocCLI(CLI): play_ds = self._play_ds(pattern, self.options.seconds, self.options.poll_interval) play = Play().load(play_ds, variable_manager=variable_manager, loader=loader) - if self.callback: + if self.callback: cb = self.callback elif self.options.one_line: cb = 'oneline' diff --git a/lib/ansible/cli/console.py b/lib/ansible/cli/console.py index 5b0432036f..d2a1d1252f 100644 --- a/lib/ansible/cli/console.py +++ b/lib/ansible/cli/console.py @@ -39,18 +39,16 @@ import sys from ansible import constants as C from ansible.cli import CLI from ansible.errors import AnsibleError - from ansible.executor.task_queue_manager import TaskQueueManager from ansible.inventory import Inventory +from ansible.module_utils._text import to_native, to_text from ansible.parsing.dataloader import DataLoader from ansible.parsing.splitter import parse_kv from ansible.playbook.play import Play -from ansible.vars import VariableManager +from ansible.plugins import module_loader from ansible.utils import module_docs from ansible.utils.color import stringc -from ansible.utils.unicode import to_unicode, to_str -from ansible.plugins import module_loader - +from ansible.vars import VariableManager try: from __main__ import display @@ -152,11 +150,11 @@ class ConsoleCLI(CLI, cmd.Cmd): continue elif module.startswith('_'): fullpath = '/'.join([path,module]) - if os.path.islink(fullpath): # avoids aliases + if os.path.islink(fullpath): # avoids aliases continue module = module.replace('_', '', 1) - module = os.path.splitext(module)[0] # removes the extension + module = os.path.splitext(module)[0] # removes the extension yield module def default(self, arg, forceshell=False): @@ -192,11 +190,11 @@ class ConsoleCLI(CLI, cmd.Cmd): ) play = Play().load(play_ds, variable_manager=self.variable_manager, loader=self.loader) except Exception as e: - display.error(u"Unable to build command: %s" % to_unicode(e)) + display.error(u"Unable to build command: %s" % to_text(e)) return False try: - cb = 'minimal' #FIXME: make callbacks configurable + cb = 'minimal' # FIXME: make callbacks configurable # now create a task queue manager to execute the play self._tqm = None try: @@ -225,8 +223,8 @@ class ConsoleCLI(CLI, cmd.Cmd): display.error('User interrupted execution') return False except Exception as e: - display.error(to_unicode(e)) - #FIXME: add traceback in very very verbose mode + display.error(to_text(e)) + # FIXME: add traceback in very very verbose mode return False def emptyline(self): @@ -379,7 +377,7 @@ class ConsoleCLI(CLI, cmd.Cmd): else: completions = [x.name for x in self.inventory.list_hosts(self.options.cwd)] - return [to_str(s)[offs:] for s in completions if to_str(s).startswith(to_str(mline))] + return [to_native(s)[offs:] for s in completions if to_native(s).startswith(to_native(mline))] def completedefault(self, text, line, begidx, endidx): if line.split()[0] in self.modules: @@ -394,7 +392,6 @@ class ConsoleCLI(CLI, cmd.Cmd): oc, a, _ = module_docs.get_docstring(in_path) return oc['options'].keys() - def run(self): super(ConsoleCLI, self).run() @@ -410,7 +407,6 @@ class ConsoleCLI(CLI, cmd.Cmd): self.pattern = self.args[0] self.options.cwd = self.pattern - # dynamically add modules as commands self.modules = self.list_modules() for module in self.modules: @@ -465,4 +461,3 @@ class ConsoleCLI(CLI, cmd.Cmd): atexit.register(readline.write_history_file, histfile) self.set_prompt() self.cmdloop() - diff --git a/lib/ansible/cli/galaxy.py b/lib/ansible/cli/galaxy.py index 231b6b909a..457e15d5c1 100644 --- a/lib/ansible/cli/galaxy.py +++ b/lib/ansible/cli/galaxy.py @@ -39,7 +39,7 @@ from ansible.galaxy.role import GalaxyRole from ansible.galaxy.login import GalaxyLogin from ansible.galaxy.token import GalaxyToken from ansible.playbook.role.requirement import RoleRequirement -from ansible.utils.unicode import to_bytes, to_unicode +from ansible.module_utils._text import to_bytes, to_text try: from __main__ import display @@ -47,6 +47,7 @@ except ImportError: from ansible.utils.display import Display display = Display() + class GalaxyCLI(CLI): SKIP_INFO_KEYS = ("name", "description", "readme_html", "related", "summary_fields", "average_aw_composite", "average_aw_score", "url" ) @@ -65,7 +66,6 @@ class GalaxyCLI(CLI): epilog = "\nSee '%s <command> --help' for more information on a specific command.\n\n" % os.path.basename(sys.argv[0]) ) - self.set_action() # common @@ -111,7 +111,7 @@ class GalaxyCLI(CLI): if self.action in ['init', 'info']: self.parser.add_option( '--offline', dest='offline', default=False, action='store_true', help="Don't query the galaxy API when creating roles") - if not self.action in ("delete","import","init","login","setup"): + if self.action not in ("delete","import","init","login","setup"): # NOTE: while the option type=str, the default is a list, and the # callback will set the value to a list. self.parser.add_option('-p', '--roles-path', dest='roles_path', action="callback", callback=CLI.expand_paths, type=str, default=C.DEFAULT_ROLES_PATH, @@ -142,7 +142,7 @@ class GalaxyCLI(CLI): def _display_role_info(self, role_info): - text = [u"", u"Role: %s" % to_unicode(role_info['name'])] + text = [u"", u"Role: %s" % to_text(role_info['name'])] text.append(u"\tdescription: %s" % role_info.get('description', '')) for k in sorted(role_info.keys()): @@ -340,7 +340,7 @@ class GalaxyCLI(CLI): f = open(role_file, 'r') if role_file.endswith('.yaml') or role_file.endswith('.yml'): try: - required_roles = yaml.safe_load(f.read()) + required_roles = yaml.safe_load(f.read()) except Exception as e: raise AnsibleError("Unable to load data from the requirements file: %s" % role_file) @@ -502,7 +502,7 @@ class GalaxyCLI(CLI): if len(self.args): terms = [] for i in range(len(self.args)): - terms.append(self.args.pop()) + terms.append(self.args.pop()) search = '+'.join(terms[::-1]) if not search and not self.options.platforms and not self.options.tags and not self.options.author: @@ -578,8 +578,8 @@ class GalaxyCLI(CLI): if len(self.args) < 2: raise AnsibleError("Expected a github_username and github_repository. Use --help.") - github_repo = self.args.pop() - github_user = self.args.pop() + github_repo = to_text(self.args.pop(), errors='surrogate_or_strict') + github_user = to_text(self.args.pop(), errors='surrogate_or_strict') if self.options.check_status: task = self.api.get_import_task(github_user=github_user, github_repo=github_repo) @@ -594,7 +594,8 @@ class GalaxyCLI(CLI): display.display("The following Galaxy roles are being updated:" + u'\n', color=C.COLOR_CHANGED) for t in task: display.display('%s.%s' % (t['summary_fields']['role']['namespace'],t['summary_fields']['role']['name']), color=C.COLOR_CHANGED) - display.display(u'\n' + "To properly namespace this role, remove each of the above and re-import %s/%s from scratch" % (github_user,github_repo), color=C.COLOR_CHANGED) + display.display(u'\nTo properly namespace this role, remove each of the above and re-import %s/%s from scratch' % (github_user, github_repo), + color=C.COLOR_CHANGED) return 0 # found a single role as expected display.display("Successfully submitted import request %d" % task[0]['id']) diff --git a/lib/ansible/cli/vault.py b/lib/ansible/cli/vault.py index 85bdac6993..65657ca70e 100644 --- a/lib/ansible/cli/vault.py +++ b/lib/ansible/cli/vault.py @@ -26,7 +26,7 @@ from ansible.errors import AnsibleError, AnsibleOptionsError from ansible.parsing.dataloader import DataLoader from ansible.parsing.vault import VaultEditor from ansible.cli import CLI -from ansible.utils.unicode import to_unicode +from ansible.module_utils._text import to_text try: from __main__ import display @@ -163,7 +163,7 @@ class VaultCLI(CLI): # unicode here because we are displaying it and therefore can make # the decision that the display doesn't have to be precisely what # the input was (leave that to decrypt instead) - self.pager(to_unicode(self.editor.plaintext(f))) + self.pager(ansible.module_utils._text.to_text(self.editor.plaintext(f))) def execute_rekey(self): for f in self.args: diff --git a/lib/ansible/errors/__init__.py b/lib/ansible/errors/__init__.py index 7f02a44bc0..2186e6bc74 100644 --- a/lib/ansible/errors/__init__.py +++ b/lib/ansible/errors/__init__.py @@ -25,8 +25,7 @@ from ansible.errors.yaml_strings import ( YAML_POSITION_DETAILS, YAML_COMMON_UNQUOTED_COLON_ERROR, YAML_COMMON_PARTIALLY_QUOTED_LINE_ERROR, YAML_COMMON_UNBALANCED_QUOTES_ERROR ) - -from ansible.utils.unicode import to_unicode, to_str +from ansible.module_utils._text import to_native, to_text class AnsibleError(Exception): @@ -54,11 +53,11 @@ class AnsibleError(Exception): if obj and isinstance(obj, AnsibleBaseYAMLObject): extended_error = self._get_extended_error() if extended_error and not suppress_extended_error: - self.message = '%s\n\n%s' % (to_str(message), to_str(extended_error)) + self.message = '%s\n\n%s' % (to_native(message), to_native(extended_error)) else: - self.message = '%s' % to_str(message) + self.message = '%s' % to_native(message) else: - self.message = '%s' % to_str(message) + self.message = '%s' % to_native(message) def __str__(self): return self.message @@ -104,8 +103,8 @@ class AnsibleError(Exception): error_message += YAML_POSITION_DETAILS % (src_file, line_number, col_number) if src_file not in ('<string>', '<unicode>') and self._show_content: (target_line, prev_line) = self._get_error_lines_from_file(src_file, line_number - 1) - target_line = to_unicode(target_line) - prev_line = to_unicode(prev_line) + target_line = to_text(target_line) + prev_line = to_text(prev_line) if target_line: stripped_line = target_line.replace(" ","") arrow_line = (" " * (col_number-1)) + "^ here" diff --git a/lib/ansible/executor/module_common.py b/lib/ansible/executor/module_common.py index 64a6d88283..86549ea0b7 100644 --- a/lib/ansible/executor/module_common.py +++ b/lib/ansible/executor/module_common.py @@ -29,11 +29,10 @@ import shlex import zipfile from io import BytesIO -# from Ansible from ansible.release import __version__, __author__ from ansible import constants as C from ansible.errors import AnsibleError -from ansible.utils.unicode import to_bytes, to_unicode +from ansible.module_utils._text import to_bytes, to_text # Must import strategy and use write_locks from there # If we import write_locks directly then we end up binding a # variable to the object and then it never gets updated. @@ -45,6 +44,7 @@ except ImportError: from ansible.utils.display import Display display = Display() + REPLACER = b"#<<INCLUDE_ANSIBLE_MODULE_COMMON>>" REPLACER_VERSION = b"\"<<ANSIBLE_VERSION>>\"" REPLACER_COMPLEX = b"\"<<INCLUDE_ANSIBLE_MODULE_COMPLEX_ARGS>>\"" @@ -239,7 +239,9 @@ def debug(command, zipped_mod, json_params): else: os.environ['PYTHONPATH'] = basedir - p = subprocess.Popen([%(interpreter)s, script_path, args_path], env=os.environ, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + p = subprocess.Popen([%(interpreter)s, script_path, args_path], + env=os.environ, shell=False, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, stdin=subprocess.PIPE) (stdout, stderr) = p.communicate() if not isinstance(stderr, (bytes, unicode)): @@ -328,6 +330,7 @@ if __name__ == '__main__': sys.exit(exitcode) ''' + def _strip_comments(source): # Strip comments and blank lines from the wrapper buf = [] @@ -338,6 +341,7 @@ def _strip_comments(source): buf.append(line) return u'\n'.join(buf) + if C.DEFAULT_KEEP_REMOTE_FILES: # Keep comments when KEEP_REMOTE_FILES is set. That way users will see # the comments with some nice usage instructions @@ -346,6 +350,7 @@ else: # ANSIBALLZ_TEMPLATE stripped of comments for smaller over the wire size ACTIVE_ANSIBALLZ_TEMPLATE = _strip_comments(ANSIBALLZ_TEMPLATE) + class ModuleDepFinder(ast.NodeVisitor): # Caveats: # This code currently does not handle: @@ -404,6 +409,7 @@ def _slurp(path): fd.close() return data + def _get_shebang(interpreter, task_vars, args=tuple()): """ Note not stellar API: @@ -425,6 +431,7 @@ def _get_shebang(interpreter, task_vars, args=tuple()): return (shebang, interpreter) + def recursive_finder(name, data, py_module_names, py_module_cache, zf): """ Using ModuleDepFinder, make sure we have all of the module_utils files that @@ -529,11 +536,13 @@ def recursive_finder(name, data, py_module_names, py_module_cache, zf): # Save memory; the file won't have to be read again for this ansible module. del py_module_cache[py_module_file] + def _is_binary(module_data): textchars = bytearray(set([7, 8, 9, 10, 12, 13, 27]) | set(range(0x20, 0x100)) - set([0x7f])) start = module_data[:1024] return bool(start.translate(None, textchars)) + def _find_snippet_imports(module_name, module_data, module_path, module_args, task_vars, module_compression): """ Given the source of the module, convert it to a Jinja2 template to insert @@ -617,9 +626,12 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta # Create the module zip data zipoutput = BytesIO() zf = zipfile.ZipFile(zipoutput, mode='w', compression=compression_method) - ### Note: If we need to import from release.py first, - ### remember to catch all exceptions: https://github.com/ansible/ansible/issues/16523 - zf.writestr('ansible/__init__.py', b'from pkgutil import extend_path\n__path__=extend_path(__path__,__name__)\n__version__="' + to_bytes(__version__) + b'"\n__author__="' + to_bytes(__author__) + b'"\n') + # Note: If we need to import from release.py first, + # remember to catch all exceptions: https://github.com/ansible/ansible/issues/16523 + zf.writestr('ansible/__init__.py', + b'from pkgutil import extend_path\n__path__=extend_path(__path__,__name__)\n__version__="' + + to_bytes(__version__) + b'"\n__author__="' + + to_bytes(__author__) + b'"\n') zf.writestr('ansible/module_utils/__init__.py', b'from pkgutil import extend_path\n__path__=extend_path(__path__,__name__)\n') zf.writestr('ansible_module_%s.py' % module_name, module_data) @@ -655,8 +667,9 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta try: zipdata = open(cached_module_filename, 'rb').read() except IOError: - raise AnsibleError('A different worker process failed to create module file. Look at traceback for that process for debugging information.') - zipdata = to_unicode(zipdata, errors='strict') + raise AnsibleError('A different worker process failed to create module file.' + ' Look at traceback for that process for debugging information.') + zipdata = to_text(zipdata, errors='surrogate_or_strict') shebang, interpreter = _get_shebang(u'/usr/bin/python', task_vars) if shebang is None: @@ -674,7 +687,7 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta shebang=shebang, interpreter=interpreter, coding=ENCODING_STRING, - ))) + ))) module_data = output.getvalue() elif module_substyle == 'powershell': @@ -721,12 +734,11 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta # The main event -- substitute the JSON args string into the module module_data = module_data.replace(REPLACER_JSONARGS, module_args_json) - facility = b'syslog.' + to_bytes(task_vars.get('ansible_syslog_facility', C.DEFAULT_SYSLOG_FACILITY), errors='strict') + facility = b'syslog.' + to_bytes(task_vars.get('ansible_syslog_facility', C.DEFAULT_SYSLOG_FACILITY), errors='surrogate_or_strict') module_data = module_data.replace(b'syslog.LOG_USER', facility) return (module_data, module_style, shebang) -# ****************************************************************************** def modify_module(module_name, module_path, module_args, task_vars=dict(), module_compression='ZIP_STORED'): """ @@ -760,7 +772,7 @@ def modify_module(module_name, module_path, module_args, task_vars=dict(), modul (module_data, module_style, shebang) = _find_snippet_imports(module_name, module_data, module_path, module_args, task_vars, module_compression) if module_style == 'binary': - return (module_data, module_style, to_unicode(shebang, nonstring='passthru')) + return (module_data, module_style, to_text(shebang, nonstring='passthru')) elif shebang is None: lines = module_data.split(b"\n", 1) if lines[0].startswith(b"#!"): @@ -769,7 +781,7 @@ def modify_module(module_name, module_path, module_args, task_vars=dict(), modul interpreter = args[0] interpreter = to_bytes(interpreter) - new_shebang = to_bytes(_get_shebang(interpreter, task_vars, args[1:])[0], errors='strict', nonstring='passthru') + new_shebang = to_bytes(_get_shebang(interpreter, task_vars, args[1:])[0], errors='surrogate_or_strict', nonstring='passthru') if new_shebang: lines[0] = shebang = new_shebang @@ -781,6 +793,6 @@ def modify_module(module_name, module_path, module_args, task_vars=dict(), modul module_data = b"\n".join(lines) else: - shebang = to_bytes(shebang, errors='strict') + shebang = to_bytes(shebang, errors='surrogate_or_strict') - return (module_data, module_style, to_unicode(shebang, nonstring='passthru')) + return (module_data, module_style, to_text(shebang, nonstring='passthru')) diff --git a/lib/ansible/executor/playbook_executor.py b/lib/ansible/executor/playbook_executor.py index 3d6d54cbe2..058230d844 100644 --- a/lib/ansible/executor/playbook_executor.py +++ b/lib/ansible/executor/playbook_executor.py @@ -21,15 +21,13 @@ __metaclass__ = type import os -from ansible.compat.six import string_types - from ansible import constants as C from ansible.executor.task_queue_manager import TaskQueueManager +from ansible.module_utils._text import to_native, to_text from ansible.playbook import Playbook from ansible.template import Templar from ansible.utils.helpers import pct_to_int from ansible.utils.path import makedirs_safe -from ansible.utils.unicode import to_unicode, to_str try: from __main__ import display @@ -74,7 +72,7 @@ class PlaybookExecutor: pb = Playbook.load(playbook_path, variable_manager=self._variable_manager, loader=self._loader) self._inventory.set_playbook_basedir(os.path.realpath(os.path.dirname(playbook_path))) - if self._tqm is None: # we are doing a listing + if self._tqm is None: # we are doing a listing entry = {'playbook': playbook_path} entry['plays'] = [] else: @@ -84,7 +82,7 @@ class PlaybookExecutor: i = 1 plays = pb.get_plays() - display.vv(u'%d plays in %s' % (len(plays), to_unicode(playbook_path))) + display.vv(u'%d plays in %s' % (len(plays), to_text(playbook_path))) for play in plays: if play._included_path is not None: @@ -110,7 +108,7 @@ class PlaybookExecutor: if self._tqm: self._tqm.send_callback('v2_playbook_on_vars_prompt', vname, private, prompt, encrypt, confirm, salt_size, salt, default) play.vars[vname] = display.do_var_prompt(vname, private, prompt, encrypt, confirm, salt_size, salt, default) - else: # we are either in --list-<option> or syntax check + else: # we are either in --list-<option> or syntax check play.vars[vname] = default # Create a temporary copy of the play here, so we can run post_validate @@ -156,7 +154,7 @@ class PlaybookExecutor: # conditions are met, we break out, otherwise we only break out if the entire # batch failed failed_hosts_count = len(self._tqm._failed_hosts) + len(self._tqm._unreachable_hosts) - \ - (previously_failed + previously_unreachable) + (previously_failed + previously_unreachable) if len(batch) == failed_hosts_count: break_play = True @@ -173,10 +171,10 @@ class PlaybookExecutor: if break_play: break - i = i + 1 # per play + i = i + 1 # per play if entry: - entrylist.append(entry) # per playbook + entrylist.append(entry) # per playbook # send the stats callback for this playbook if self._tqm is not None: @@ -276,7 +274,7 @@ class PlaybookExecutor: for x in replay_hosts: fd.write("%s\n" % x) except Exception as e: - display.warning("Could not create retry file '%s'.\n\t%s" % (retry_path, to_str(e))) + display.warning("Could not create retry file '%s'.\n\t%s" % (retry_path, to_native(e))) return False return True diff --git a/lib/ansible/executor/process/worker.py b/lib/ansible/executor/process/worker.py index d579d1fa3c..d93de24ab3 100644 --- a/lib/ansible/executor/process/worker.py +++ b/lib/ansible/executor/process/worker.py @@ -19,16 +19,10 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type -from ansible.compat.six.moves import queue - -import json import multiprocessing import os -import signal import sys -import time import traceback -import zlib from jinja2.exceptions import TemplateNotFound @@ -40,13 +34,10 @@ try: except ImportError: HAS_ATFORK=False -from ansible.errors import AnsibleError, AnsibleConnectionFailure +from ansible.errors import AnsibleConnectionFailure from ansible.executor.task_executor import TaskExecutor from ansible.executor.task_result import TaskResult -from ansible.playbook.handler import Handler -from ansible.playbook.task import Task -from ansible.vars.unsafe_proxy import AnsibleJSONUnsafeDecoder -from ansible.utils.unicode import to_unicode +from ansible.module_utils._text import to_text try: from __main__ import display @@ -144,11 +135,11 @@ class WorkerProcess(multiprocessing.Process): try: self._host.vars = dict() self._host.groups = [] - task_result = TaskResult(self._host.name, self._task._uuid, dict(failed=True, exception=to_unicode(traceback.format_exc()), stdout='')) + task_result = TaskResult(self._host.name, self._task._uuid, dict(failed=True, exception=to_text(traceback.format_exc()), stdout='')) self._rslt_q.put(task_result, block=False) except: - display.debug(u"WORKER EXCEPTION: %s" % to_unicode(e)) - display.debug(u"WORKER TRACEBACK: %s" % to_unicode(traceback.format_exc())) + display.debug(u"WORKER EXCEPTION: %s" % to_text(e)) + display.debug(u"WORKER TRACEBACK: %s" % to_text(traceback.format_exc())) display.debug("WORKER PROCESS EXITING") diff --git a/lib/ansible/executor/task_executor.py b/lib/ansible/executor/task_executor.py index a5bb076761..91da5d3b04 100644 --- a/lib/ansible/executor/task_executor.py +++ b/lib/ansible/executor/task_executor.py @@ -20,7 +20,6 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type import base64 -import json import subprocess import sys import time @@ -31,12 +30,12 @@ from ansible.compat.six import iteritems, string_types, binary_type from ansible import constants as C from ansible.errors import AnsibleError, AnsibleParserError, AnsibleUndefinedVariable, AnsibleConnectionFailure from ansible.executor.task_result import TaskResult +from ansible.module_utils._text import to_bytes, to_text from ansible.playbook.conditional import Conditional from ansible.playbook.task import Task from ansible.template import Templar from ansible.utils.encrypt import key_for_hostname from ansible.utils.listify import listify_lookup_plugin_terms -from ansible.utils.unicode import to_unicode, to_bytes from ansible.vars.unsafe_proxy import UnsafeProxy, wrap_var try: @@ -130,7 +129,7 @@ class TaskExecutor: if isinstance(res, UnsafeProxy): return res._obj elif isinstance(res, binary_type): - return to_unicode(res, errors='strict') + return to_text(res, errors='surrogate_or_strict') elif isinstance(res, dict): for k in res: res[k] = _clean_res(res[k]) @@ -144,16 +143,16 @@ class TaskExecutor: display.debug("done dumping result, returning") return res except AnsibleError as e: - return dict(failed=True, msg=to_unicode(e, nonstring='simplerepr')) + return dict(failed=True, msg=to_text(e, nonstring='simplerepr')) except Exception as e: - return dict(failed=True, msg='Unexpected failure during module execution.', exception=to_unicode(traceback.format_exc()), stdout='') + return dict(failed=True, msg='Unexpected failure during module execution.', exception=to_text(traceback.format_exc()), stdout='') finally: try: self._connection.close() except AttributeError: pass except Exception as e: - display.debug(u"error closing connection: %s" % to_unicode(e)) + display.debug(u"error closing connection: %s" % to_text(e)) def _get_loop_items(self): ''' @@ -177,16 +176,18 @@ class TaskExecutor: items = None if self._task.loop: if self._task.loop in self._shared_loader_obj.lookup_loader: - #TODO: remove convert_bare true and deprecate this in with_ + # TODO: remove convert_bare true and deprecate this in with_ if self._task.loop == 'first_found': # first_found loops are special. If the item is undefined # then we want to fall through to the next value rather # than failing. - loop_terms = listify_lookup_plugin_terms(terms=self._task.loop_args, templar=templar, loader=self._loader, fail_on_undefined=False, convert_bare=True) + loop_terms = listify_lookup_plugin_terms(terms=self._task.loop_args, templar=templar, + loader=self._loader, fail_on_undefined=False, convert_bare=True) loop_terms = [t for t in loop_terms if not templar._contains_vars(t)] else: try: - loop_terms = listify_lookup_plugin_terms(terms=self._task.loop_args, templar=templar, loader=self._loader, fail_on_undefined=True, convert_bare=True) + loop_terms = listify_lookup_plugin_terms(terms=self._task.loop_args, templar=templar, + loader=self._loader, fail_on_undefined=True, convert_bare=True) except AnsibleUndefinedVariable as e: display.deprecated("Skipping task due to undefined Error, in the future this will be a fatal error.: %s" % to_bytes(e)) return None @@ -195,7 +196,7 @@ class TaskExecutor: mylookup = self._shared_loader_obj.lookup_loader.get(self._task.loop, loader=self._loader, templar=templar) # give lookup task 'context' for subdir (mostly needed for first_found) - for subdir in ['template', 'var', 'file']: #TODO: move this to constants? + for subdir in ['template', 'var', 'file']: # TODO: move this to constants? if subdir in self._task.action: break setattr(mylookup,'_subdir', subdir + 's') @@ -239,13 +240,15 @@ class TaskExecutor: label = None loop_pause = 0 if self._task.loop_control: - # the value may be 'None', so we still need to default it back to 'item' + # the value may be 'None', so we still need to default it back to 'item' loop_var = self._task.loop_control.loop_var or 'item' label = self._task.loop_control.label or ('{{' + loop_var + '}}') loop_pause = self._task.loop_control.pause or 0 if loop_var in task_vars: - display.warning("The loop variable '%s' is already in use. You should set the `loop_var` value in the `loop_control` option for the task to something else to avoid variable collisions and unexpected behavior." % loop_var) + display.warning(u"The loop variable '%s' is already in use." + u"You should set the `loop_var` value in the `loop_control` option for the task" + u" to something else to avoid variable collisions and unexpected behavior." % loop_var) ran_once = False items = self._squash_items(items, loop_var, task_vars) @@ -263,7 +266,7 @@ class TaskExecutor: tmp_task._parent = self._task._parent tmp_play_context = self._play_context.copy() except AnsibleParserError as e: - results.append(dict(failed=True, msg=to_unicode(e))) + results.append(dict(failed=True, msg=to_text(e))) continue # now we swap the internal task and play context with their copies, @@ -279,7 +282,7 @@ class TaskExecutor: res[loop_var] = item res['_ansible_item_result'] = True - if not label is None: + if label is not None: templar = Templar(loader=self._loader, shared_loader_obj=self._shared_loader_obj, variables=self._job_vars) res['_ansible_item_label'] = templar.template(label, fail_on_undefined=False) @@ -421,7 +424,7 @@ class TaskExecutor: include_file = templar.template(include_file) return dict(include=include_file, include_variables=include_variables) - #TODO: not needed? + # TODO: not needed? # if this task is a IncludeRole, we just return now with a success code so the main thread can expand the task list for the given host elif self._task.action == 'include_role': include_variables = self._task.args.copy() @@ -482,7 +485,7 @@ class TaskExecutor: try: result = self._handler.run(task_vars=variables) except AnsibleConnectionFailure as e: - return dict(unreachable=True, msg=to_unicode(e)) + return dict(unreachable=True, msg=to_text(e)) display.debug("handler run complete") # preserve no log @@ -666,7 +669,7 @@ class TaskExecutor: try: cmd = subprocess.Popen(['ssh','-o','ControlPersist'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) (out, err) = cmd.communicate() - err = to_unicode(err) + err = to_text(err) if u"Bad configuration option" in err or u"Usage:" in err: conn_type = "paramiko" except OSError: diff --git a/lib/ansible/executor/task_queue_manager.py b/lib/ansible/executor/task_queue_manager.py index b1a205d2e5..6744367363 100644 --- a/lib/ansible/executor/task_queue_manager.py +++ b/lib/ansible/executor/task_queue_manager.py @@ -28,21 +28,20 @@ import time from collections import deque from ansible import constants as C +from ansible.compat.six import string_types from ansible.errors import AnsibleError from ansible.executor import action_write_locks from ansible.executor.play_iterator import PlayIterator from ansible.executor.process.worker import WorkerProcess from ansible.executor.stats import AggregateStats -from ansible.module_utils.facts import Facts +from ansible.module_utils._text import to_text from ansible.playbook.block import Block from ansible.playbook.play_context import PlayContext from ansible.plugins import action_loader, callback_loader, connection_loader, filter_loader, lookup_loader, module_loader, strategy_loader, test_loader -from ansible.template import Templar -from ansible.vars.hostvars import HostVars from ansible.plugins.callback import CallbackBase +from ansible.template import Templar from ansible.utils.helpers import pct_to_int -from ansible.utils.unicode import to_unicode -from ansible.compat.six import string_types +from ansible.vars.hostvars import HostVars try: from __main__ import display @@ -288,7 +287,8 @@ class TaskQueueManager: stdout_callback_loaded = True elif callback_name == 'tree' and self._run_tree: pass - elif not self._run_additional_callbacks or (callback_needs_whitelist and (C.DEFAULT_CALLBACK_WHITELIST is None or callback_name not in C.DEFAULT_CALLBACK_WHITELIST)): + elif not self._run_additional_callbacks or (callback_needs_whitelist and ( + C.DEFAULT_CALLBACK_WHITELIST is None or callback_name not in C.DEFAULT_CALLBACK_WHITELIST)): continue self._callback_plugins.append(callback_plugin()) @@ -336,8 +336,8 @@ class TaskQueueManager: serial_items = [serial_items] max_serial = max([pct_to_int(x, num_hosts) for x in serial_items]) - contenders = [self._options.forks, max_serial, num_hosts] - contenders = [v for v in contenders if v is not None and v > 0] + contenders = [self._options.forks, max_serial, num_hosts] + contenders = [v for v in contenders if v is not None and v > 0] self._initialize_processes(min(contenders)) play_context = PlayContext(new_play, self._options, self.passwords, self._connection_lockfile.fileno()) @@ -446,7 +446,7 @@ class TaskQueueManager: # try to find v2 method, fallback to v1 method, ignore callback if no method found methods = [] for possible in [method_name, 'v2_on_any']: - gotit = getattr(callback_plugin, possible, None) + gotit = getattr(callback_plugin, possible, None) if gotit is None: gotit = getattr(callback_plugin, possible.replace('v2_',''), None) if gotit is not None: @@ -468,8 +468,8 @@ class TaskQueueManager: else: method(*args, **kwargs) except Exception as e: - #TODO: add config toggle to make this fatal or not? - display.warning(u"Failure using method (%s) in callback plugin (%s): %s" % (to_unicode(method_name), to_unicode(callback_plugin), to_unicode(e))) + # TODO: add config toggle to make this fatal or not? + display.warning(u"Failure using method (%s) in callback plugin (%s): %s" % (to_text(method_name), to_text(callback_plugin), to_text(e))) from traceback import format_tb from sys import exc_info display.debug('Callback Exception: \n' + ' '.join(format_tb(exc_info()[2]))) diff --git a/lib/ansible/galaxy/api.py b/lib/ansible/galaxy/api.py index 5dd9d75736..471c8eb4e9 100644 --- a/lib/ansible/galaxy/api.py +++ b/lib/ansible/galaxy/api.py @@ -28,12 +28,12 @@ import json import ansible.constants as C from ansible.compat.six import string_types -from ansible.compat.six.moves.urllib.parse import quote as urlquote, urlencode from ansible.compat.six.moves.urllib.error import HTTPError +from ansible.compat.six.moves.urllib.parse import quote as urlquote, urlencode from ansible.errors import AnsibleError -from ansible.module_utils.urls import open_url from ansible.galaxy.token import GalaxyToken -from ansible.utils.unicode import to_str +from ansible.module_utils._text import to_native +from ansible.module_utils.urls import open_url try: from __main__ import display @@ -115,12 +115,12 @@ class GalaxyAPI(object): try: return_data = open_url(url, validate_certs=self._validate_certs) except Exception as e: - raise AnsibleError("Failed to get data from the API server (%s): %s " % (url, to_str(e))) + raise AnsibleError("Failed to get data from the API server (%s): %s " % (url, to_native(e))) try: data = json.load(return_data) except Exception as e: - raise AnsibleError("Could not process data from the API server (%s): %s " % (url, to_str(e))) + raise AnsibleError("Could not process data from the API server (%s): %s " % (url, to_native(e))) if 'current_version' not in data: raise AnsibleError("missing required 'current_version' from server response (%s)" % url) diff --git a/lib/ansible/inventory/__init__.py b/lib/ansible/inventory/__init__.py index f8a98929f4..eb498bc238 100644 --- a/lib/ansible/inventory/__init__.py +++ b/lib/ansible/inventory/__init__.py @@ -33,12 +33,10 @@ from ansible.errors import AnsibleError from ansible.inventory.dir import InventoryDirectory, get_file_parser from ansible.inventory.group import Group from ansible.inventory.host import Host +from ansible.module_utils._text import to_bytes, to_text +from ansible.parsing.utils.addresses import parse_address from ansible.plugins import vars_loader -from ansible.utils.unicode import to_unicode, to_bytes from ansible.utils.vars import combine_vars -from ansible.parsing.utils.addresses import parse_address - -HOSTS_PATTERNS_CACHE = {} try: from __main__ import display @@ -46,6 +44,10 @@ except ImportError: from ansible.utils.display import Display display = Display() + +HOSTS_PATTERNS_CACHE = {} + + class Inventory(object): """ Host inventory for ansible. @@ -125,7 +127,7 @@ class Inventory(object): try: (host, port) = parse_address(h, allow_ranges=False) except AnsibleError as e: - display.vvv("Unable to parse address from hostname, leaving unchanged: %s" % to_unicode(e)) + display.vvv("Unable to parse address from hostname, leaving unchanged: %s" % to_text(e)) host = h port = None @@ -138,7 +140,7 @@ class Inventory(object): self.localhost = new_host all.add_host(new_host) elif self._loader.path_exists(host_list): - #TODO: switch this to a plugin loader and a 'condition' per plugin on which it should be tried, restoring 'inventory pllugins' + # TODO: switch this to a plugin loader and a 'condition' per plugin on which it should be tried, restoring 'inventory pllugins' if self.is_directory(host_list): # Ensure basedir is inside the directory host_list = os.path.join(self.host_list, "") @@ -151,7 +153,7 @@ class Inventory(object): # should never happen, but JIC raise AnsibleError("Unable to parse %s as an inventory source" % host_list) else: - display.warning("Host file not found: %s" % to_unicode(host_list)) + display.warning("Host file not found: %s" % to_text(host_list)) self._vars_plugins = [ x for x in vars_loader.all(self) ] @@ -191,7 +193,7 @@ class Inventory(object): return results def get_hosts(self, pattern="all", ignore_limits_and_restrictions=False): - """ + """ Takes a pattern or list of patterns and returns a list of matching inventory host names, taking into account any active restrictions or applied subsets @@ -205,9 +207,9 @@ class Inventory(object): if not ignore_limits_and_restrictions: if self._subset: - pattern_hash += u":%s" % to_unicode(self._subset) + pattern_hash += u":%s" % to_text(self._subset) if self._restriction: - pattern_hash += u":%s" % to_unicode(self._restriction) + pattern_hash += u":%s" % to_text(self._restriction) if pattern_hash not in HOSTS_PATTERNS_CACHE: @@ -326,7 +328,7 @@ class Inventory(object): return hosts def _match_one_pattern(self, pattern): - """ + """ Takes a single pattern and returns a list of matching host names. Ignores intersection (&) and exclusion (!) specifiers. @@ -426,7 +428,7 @@ class Inventory(object): """ Takes a list of hosts and a (start,end) tuple and returns the subset of hosts based on the subscript (which may be None to return all hosts). - """ + """ if not hosts or not subscript: return hosts @@ -491,7 +493,8 @@ class Inventory(object): py_interp = sys.executable if not py_interp: # sys.executable is not set in some cornercases. #13585 - display.warning('Unable to determine python interpreter from sys.executable. Using /usr/bin/python default. You can correct this by setting ansible_python_interpreter for localhost') + display.warning('Unable to determine python interpreter from sys.executable. Using /usr/bin/python default.' + ' You can correct this by setting ansible_python_interpreter for localhost') py_interp = '/usr/bin/python' new_host.set_variable("ansible_python_interpreter", py_interp) self.get_group("ungrouped").add_host(new_host) @@ -648,7 +651,7 @@ class Inventory(object): return sorted(self.groups.keys(), key=lambda x: x) def restrict_to_hosts(self, restriction): - """ + """ Restrict list operations to the hosts given in restriction. This is used to batch serial operations in main playbook code, don't use this for other reasons. @@ -660,12 +663,12 @@ class Inventory(object): self._restriction = [ h.name for h in restriction ] def subset(self, subset_pattern): - """ + """ Limits inventory results to a subset of inventory that matches a given pattern, such as to select a given geographic of numeric slice amongst - a previous 'hosts' selection that only select roles, or vice versa. + a previous 'hosts' selection that only select roles, or vice versa. Corresponds to --limit parameter to ansible-playbook - """ + """ if subset_pattern is None: self._subset = None else: @@ -781,7 +784,7 @@ class Inventory(object): path = os.path.realpath(os.path.join(basedir, 'group_vars')) found_vars = set() if os.path.exists(path): - found_vars = set(os.listdir(to_unicode(path))) + found_vars = set(os.listdir(to_text(path))) return found_vars def _find_host_vars_files(self, basedir): @@ -791,7 +794,7 @@ class Inventory(object): path = os.path.realpath(os.path.join(basedir, 'host_vars')) found_vars = set() if os.path.exists(path): - found_vars = set(os.listdir(to_unicode(path))) + found_vars = set(os.listdir(to_text(path))) return found_vars def _get_hostgroup_vars(self, host=None, group=None, new_pb_basedir=False, return_results=False): @@ -832,13 +835,13 @@ class Inventory(object): # Before trying to load vars from file, check that the directory contains relvant file names if host is None and any(map(lambda ext: group.name + ext in self._group_vars_files, C.YAML_FILENAME_EXTENSIONS)): # load vars in dir/group_vars/name_of_group - base_path = to_unicode(os.path.abspath(os.path.join(to_bytes(basedir), b"group_vars/" + to_bytes(group.name))), errors='strict') + base_path = to_text(os.path.abspath(os.path.join(to_bytes(basedir), b"group_vars/" + to_bytes(group.name))), errors='surrogate_or_strict') host_results = self._variable_manager.add_group_vars_file(base_path, self._loader) if return_results: results = combine_vars(results, host_results) elif group is None and any(map(lambda ext: host.name + ext in self._host_vars_files, C.YAML_FILENAME_EXTENSIONS)): # same for hostvars in dir/host_vars/name_of_host - base_path = to_unicode(os.path.abspath(os.path.join(to_bytes(basedir), b"host_vars/" + to_bytes(host.name))), errors='strict') + base_path = to_text(os.path.abspath(os.path.join(to_bytes(basedir), b"host_vars/" + to_bytes(host.name))), errors='surrogate_or_strict') group_results = self._variable_manager.add_host_vars_file(base_path, self._loader) if return_results: results = combine_vars(results, group_results) diff --git a/lib/ansible/inventory/ini.py b/lib/ansible/inventory/ini.py index 172f8d0351..81c3ec90e4 100644 --- a/lib/ansible/inventory/ini.py +++ b/lib/ansible/inventory/ini.py @@ -28,9 +28,10 @@ from ansible.inventory.host import Host from ansible.inventory.group import Group from ansible.inventory.expand_hosts import detect_range from ansible.inventory.expand_hosts import expand_hostname_range +from ansible.module_utils._text import to_text from ansible.parsing.utils.addresses import parse_address from ansible.utils.shlex import shlex_split -from ansible.utils.unicode import to_unicode + class InventoryParser(object): """ @@ -56,7 +57,7 @@ class InventoryParser(object): (data, private) = loader._get_file_contents(filename) else: with open(filename) as fh: - data = to_unicode(fh.read()) + data = to_text(fh.read()) data = data.split('\n') self._parse(data) @@ -125,7 +126,7 @@ class InventoryParser(object): continue elif line.startswith('[') and line.endswith(']'): - self._raise_error("Invalid section entry: '%s'. Please make sure that there are no spaces" % line + \ + self._raise_error("Invalid section entry: '%s'. Please make sure that there are no spaces" % line + "in the section entry, and that there are no other invalid characters") # It's not a section, so the current state tells us what kind of @@ -188,7 +189,6 @@ class InventoryParser(object): if group.depth == 0 and group.name not in ('all', 'ungrouped'): self.groups['all'].add_child_group(group) - def _parse_group_name(self, line): ''' Takes a single line and tries to parse it as a group name. Returns the @@ -323,7 +323,7 @@ class InventoryParser(object): except SyntaxError: # Is this a hash with an equals at the end? pass - return to_unicode(v, nonstring='passthru', errors='strict') + return to_text(v, nonstring='passthru', errors='surrogate_or_strict') def get_host_variables(self, host): return {} diff --git a/lib/ansible/inventory/script.py b/lib/ansible/inventory/script.py index 95e48eff58..6288b79445 100644 --- a/lib/ansible/inventory/script.py +++ b/lib/ansible/inventory/script.py @@ -31,7 +31,7 @@ from ansible.errors import AnsibleError from ansible.inventory.host import Host from ansible.inventory.group import Group from ansible.module_utils.basic import json_dict_bytes_to_unicode -from ansible.utils.unicode import to_str, to_unicode +from ansible.module_utils._text import to_native, to_text class InventoryScript: @@ -61,9 +61,9 @@ class InventoryScript: # make sure script output is unicode so that json loader will output # unicode strings itself try: - self.data = to_unicode(stdout, errors="strict") + self.data = to_text(stdout, errors="strict") except Exception as e: - raise AnsibleError("inventory data from {0} contained characters that cannot be interpreted as UTF-8: {1}".format(to_str(self.filename), to_str(e))) + raise AnsibleError("inventory data from {0} contained characters that cannot be interpreted as UTF-8: {1}".format(to_native(self.filename), to_native(e))) # see comment about _meta below self.host_vars_from_top = None @@ -78,11 +78,11 @@ class InventoryScript: self.raw = self._loader.load(self.data) except Exception as e: sys.stderr.write(err + "\n") - raise AnsibleError("failed to parse executable inventory script results from {0}: {1}".format(to_str(self.filename), to_str(e))) + raise AnsibleError("failed to parse executable inventory script results from {0}: {1}".format(to_native(self.filename), to_native(e))) if not isinstance(self.raw, Mapping): sys.stderr.write(err + "\n") - raise AnsibleError("failed to parse executable inventory script results from {0}: data needs to be formatted as a json dict".format(to_str(self.filename))) + raise AnsibleError("failed to parse executable inventory script results from {0}: data needs to be formatted as a json dict".format(to_native(self.filename))) group = None for (group_name, data) in self.raw.items(): @@ -152,7 +152,7 @@ class InventoryScript: try: got = self.host_vars_from_top.get(host.name, {}) except AttributeError as e: - raise AnsibleError("Improperly formated host information for %s: %s" % (host.name,to_str(e))) + raise AnsibleError("Improperly formated host information for %s: %s" % (host.name,to_native(e))) return got cmd = [self.filename, "--host", host.name] diff --git a/lib/ansible/module_utils/_text.py b/lib/ansible/module_utils/_text.py index fe5acf7f27..cf075e3bb4 100644 --- a/lib/ansible/module_utils/_text.py +++ b/lib/ansible/module_utils/_text.py @@ -32,16 +32,18 @@ making backwards compatibility guarantees. The API may change between releases. Do not use this unless you are willing to port your module code. """ +import codecs from ansible.module_utils.six import PY3, text_type, binary_type -import codecs + try: codecs.lookup_error('surrogateescape') HAS_SURROGATEESCAPE = True except LookupError: HAS_SURROGATEESCAPE = False + def to_bytes(obj, encoding='utf-8', errors=None, nonstring='simplerepr'): """Make sure that a string is a byte string @@ -109,7 +111,14 @@ def to_bytes(obj, encoding='utf-8', errors=None, nonstring='simplerepr'): # Note: We do these last even though we have to call to_bytes again on the # value because we're optimizing the common case if nonstring == 'simplerepr': - value = str(obj) + try: + value = str(obj) + except UnicodeError: + try: + value = repr(obj) + except UnicodeError: + # Giving up + return to_bytes('') elif nonstring == 'passthru': return obj elif nonstring == 'empty': @@ -122,6 +131,7 @@ def to_bytes(obj, encoding='utf-8', errors=None, nonstring='simplerepr'): return to_bytes(value, encoding, errors) + def to_text(obj, encoding='utf-8', errors=None, nonstring='simplerepr'): """Make sure that a string is a text string @@ -175,7 +185,14 @@ def to_text(obj, encoding='utf-8', errors=None, nonstring='simplerepr'): # Note: We do these last even though we have to call to_text again on the # value because we're optimizing the common case if nonstring == 'simplerepr': - value = str(obj) + try: + value = str(obj) + except UnicodeError: + try: + value = repr(obj) + except UnicodeError: + # Giving up + return u'' elif nonstring == 'passthru': return obj elif nonstring == 'empty': @@ -187,6 +204,7 @@ def to_text(obj, encoding='utf-8', errors=None, nonstring='simplerepr'): return to_text(value, encoding, errors) + #: :py:func:`to_native` #: Transform a variable into the native str type for the python version #: diff --git a/lib/ansible/module_utils/basic.py b/lib/ansible/module_utils/basic.py index 12add4eb38..61f68a3965 100644 --- a/lib/ansible/module_utils/basic.py +++ b/lib/ansible/module_utils/basic.py @@ -805,7 +805,7 @@ class AnsibleModule(object): if not HAVE_SELINUX or not self.selinux_enabled(): return context try: - ret = selinux.matchpathcon(to_native(path, errors='strict'), mode) + ret = selinux.matchpathcon(to_native(path, errors='surrogate_or_strict'), mode) except OSError: return context if ret[0] == -1: @@ -820,7 +820,7 @@ class AnsibleModule(object): if not HAVE_SELINUX or not self.selinux_enabled(): return context try: - ret = selinux.lgetfilecon_raw(to_native(path, errors='strict')) + ret = selinux.lgetfilecon_raw(to_native(path, errors='surrogate_or_strict')) except OSError: e = get_exception() if e.errno == errno.ENOENT: @@ -2121,10 +2121,10 @@ class AnsibleModule(object): to_clean_args = args if PY2: if isinstance(args, text_type): - to_clean_args = args.encode('utf-8') + to_clean_args = to_bytes(args) else: if isinstance(args, binary_type): - to_clean_args = args.decode('utf-8', errors='replace') + to_clean_args = to_text(args) if isinstance(args, (text_type, binary_type)): to_clean_args = shlex.split(to_clean_args) @@ -2193,11 +2193,7 @@ class AnsibleModule(object): if not binary_data: data += '\n' if isinstance(data, text_type): - if PY3: - errors = 'surrogateescape' - else: - errors = 'strict' - data = data.encode('utf-8', errors=errors) + data = to_bytes(data) cmd.stdin.write(data) cmd.stdin.close() diff --git a/lib/ansible/parsing/dataloader.py b/lib/ansible/parsing/dataloader.py index 9cac94d83c..f27e31ae2b 100644 --- a/lib/ansible/parsing/dataloader.py +++ b/lib/ansible/parsing/dataloader.py @@ -27,16 +27,15 @@ import tempfile from yaml import YAMLError from ansible.compat.six import text_type, string_types - from ansible.errors import AnsibleFileNotFound, AnsibleParserError, AnsibleError from ansible.errors.yaml_strings import YAML_SYNTAX_ERROR +from ansible.module_utils.basic import is_executable +from ansible.module_utils._text import to_bytes, to_native, to_text from ansible.parsing.vault import VaultLib from ansible.parsing.quoting import unquote from ansible.parsing.yaml.loader import AnsibleLoader from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleUnicode -from ansible.module_utils.basic import is_executable from ansible.utils.path import unfrackpath -from ansible.utils.unicode import to_unicode, to_bytes, to_str try: from __main__ import display @@ -44,6 +43,7 @@ except ImportError: from ansible.utils.display import Display display = Display() + class DataLoader(): ''' @@ -127,15 +127,15 @@ class DataLoader(): def path_exists(self, path): path = self.path_dwim(path) - return os.path.exists(to_bytes(path, errors='strict')) + return os.path.exists(to_bytes(path, errors='surrogate_or_strict')) def is_file(self, path): path = self.path_dwim(path) - return os.path.isfile(to_bytes(path, errors='strict')) or path == os.devnull + return os.path.isfile(to_bytes(path, errors='surrogate_or_strict')) or path == os.devnull def is_directory(self, path): path = self.path_dwim(path) - return os.path.isdir(to_bytes(path, errors='strict')) + return os.path.isdir(to_bytes(path, errors='surrogate_or_strict')) def list_directory(self, path): path = self.path_dwim(path) @@ -156,7 +156,7 @@ class DataLoader(): try: loader.dispose() except AttributeError: - pass # older versions of yaml don't have dispose function, ignore + pass # older versions of yaml don't have dispose function, ignore def _get_file_contents(self, file_name): ''' @@ -178,7 +178,7 @@ class DataLoader(): data = self._vault.decrypt(data, filename=b_file_name) show_content = False - data = to_unicode(data, errors='strict') + data = to_text(data, errors='surrogate_or_strict') return (data, show_content) except (IOError, OSError) as e: @@ -208,7 +208,7 @@ class DataLoader(): ''' sets the base directory, used to find files when a relative path is given ''' if basedir is not None: - self._basedir = to_unicode(basedir) + self._basedir = to_text(basedir) def path_dwim(self, given): ''' @@ -216,14 +216,14 @@ class DataLoader(): ''' given = unquote(given) - given = to_unicode(given, errors='strict') + given = to_text(given, errors='surrogate_or_strict') if given.startswith(u"/"): return os.path.abspath(given) elif given.startswith(u"~"): return os.path.abspath(os.path.expanduser(given)) else: - basedir = to_unicode(self._basedir, errors='strict') + basedir = to_text(self._basedir, errors='surrogate_or_strict') return os.path.abspath(os.path.join(basedir, given)) def path_dwim_relative(self, path, dirname, source): @@ -247,8 +247,8 @@ class DataLoader(): basedir = unfrackpath(path) # is it a role and if so make sure you get correct base path - if path.endswith('tasks') and os.path.exists(to_bytes(os.path.join(path,'main.yml'), errors='strict')) \ - or os.path.exists(to_bytes(os.path.join(path,'tasks/main.yml'), errors='strict')): + if path.endswith('tasks') and os.path.exists(to_bytes(os.path.join(path,'main.yml'), errors='surrogate_or_strict')) \ + or os.path.exists(to_bytes(os.path.join(path,'tasks/main.yml'), errors='surrogate_or_strict')): isrole = True if path.endswith('tasks'): basedir = unfrackpath(os.path.dirname(path)) @@ -271,7 +271,7 @@ class DataLoader(): search.append(self.path_dwim(source)) for candidate in search: - if os.path.exists(to_bytes(candidate, errors='strict')): + if os.path.exists(to_bytes(candidate, errors='surrogate_or_strict')): break return candidate @@ -296,19 +296,19 @@ class DataLoader(): elif source.startswith('~') or source.startswith(os.path.sep): # path is absolute, no relative needed, check existence and return source test_path = unfrackpath(b_source) - if os.path.exists(to_bytes(test_path, errors='strict')): + if os.path.exists(to_bytes(test_path, errors='surrogate_or_strict')): result = test_path else: search = [] for path in paths: upath = unfrackpath(path) - b_upath = to_bytes(upath, errors='strict') + b_upath = to_bytes(upath, errors='surrogate_or_strict') b_mydir = os.path.dirname(b_upath) # if path is in role and 'tasks' not there already, add it into the search if b_upath.endswith(b'tasks') and os.path.exists(os.path.join(b_upath, b'main.yml')) \ - or os.path.exists(os.path.join(b_upath, b'tasks/main.yml')) \ - or os.path.exists(os.path.join(b_mydir, b'tasks/main.yml')): + or os.path.exists(os.path.join(b_upath, b'tasks/main.yml')) \ + or os.path.exists(os.path.join(b_mydir, b'tasks/main.yml')): if b_mydir.endswith(b'tasks'): search.append(os.path.join(os.path.dirname(b_mydir), b_dirname, b_source)) search.append(os.path.join(b_mydir, b_source)) @@ -324,11 +324,11 @@ class DataLoader(): search.append(os.path.join(to_bytes(self.get_basedir()), b_dirname, b_source)) search.append(os.path.join(to_bytes(self.get_basedir()), b_source)) - display.debug(u'search_path:\n\t%s' % to_unicode(b'\n\t'.join(search), errors='replace')) + display.debug(u'search_path:\n\t%s' % to_text(b'\n\t'.join(search))) for b_candidate in search: - display.vvvvv(u'looking for "%s" at "%s"' % (source, to_unicode(b_candidate))) + display.vvvvv(u'looking for "%s" at "%s"' % (source, to_text(b_candidate))) if os.path.exists(b_candidate): - result = to_unicode(b_candidate) + result = to_text(b_candidate) break return result @@ -339,8 +339,8 @@ class DataLoader(): retrieve password from STDOUT """ - this_path = os.path.realpath(to_bytes(os.path.expanduser(vault_password_file), errors='strict')) - if not os.path.exists(to_bytes(this_path, errors='strict')): + this_path = os.path.realpath(to_bytes(os.path.expanduser(vault_password_file), errors='surrogate_or_strict')) + if not os.path.exists(to_bytes(this_path, errors='surrogate_or_strict')): raise AnsibleFileNotFound("The vault password file %s was not found" % this_path) if self.is_executable(this_path): @@ -348,7 +348,8 @@ class DataLoader(): # STDERR not captured to make it easier for users to prompt for input in their scripts p = subprocess.Popen(this_path, stdout=subprocess.PIPE) except OSError as e: - raise AnsibleError("Problem running vault password script %s (%s). If this is not a script, remove the executable bit from the file." % (' '.join(this_path), e)) + raise AnsibleError("Problem running vault password script %s (%s)." + " If this is not a script, remove the executable bit from the file." % (' '.join(this_path), to_native(e))) stdout, stderr = p.communicate() self.set_vault_password(stdout.strip('\r\n')) else: @@ -381,11 +382,11 @@ class DataLoader(): """ if not file_path or not isinstance(file_path, string_types): - raise AnsibleParserError("Invalid filename: '%s'" % to_str(file_path)) + raise AnsibleParserError("Invalid filename: '%s'" % to_native(file_path)) - b_file_path = to_bytes(file_path, errors='strict') + b_file_path = to_bytes(file_path, errors='surrogate_or_strict') if not self.path_exists(b_file_path) or not self.is_file(b_file_path): - raise AnsibleFileNotFound("the file_name '%s' does not exist, or is not readable" % to_str(file_path)) + raise AnsibleFileNotFound("the file_name '%s' does not exist, or is not readable" % to_native(file_path)) if not self._vault: self._vault = VaultLib(password="") @@ -410,7 +411,7 @@ class DataLoader(): return real_path except (IOError, OSError) as e: - raise AnsibleParserError("an error occurred while trying to read the file '%s': %s" % (to_str(real_path), to_str(e))) + raise AnsibleParserError("an error occurred while trying to read the file '%s': %s" % (to_native(real_path), to_native(e))) def cleanup_tmp_file(self, file_path): """ @@ -420,11 +421,11 @@ class DataLoader(): """ if file_path in self._tempfiles: os.unlink(file_path) - self._tempfiles.remove(file_path); + self._tempfiles.remove(file_path) def cleanup_all_tmp_files(self): for f in self._tempfiles: try: self.cleanup_tmp_file(f) except: - pass #TODO: this should at least warn + pass # TODO: this should at least warn diff --git a/lib/ansible/parsing/splitter.py b/lib/ansible/parsing/splitter.py index 4849d08265..e6cb452035 100644 --- a/lib/ansible/parsing/splitter.py +++ b/lib/ansible/parsing/splitter.py @@ -23,8 +23,10 @@ import re import codecs from ansible.errors import AnsibleParserError +from ansible.module_utils._text import to_text from ansible.parsing.quoting import unquote + # Decode escapes adapted from rspeer's answer here: # http://stackoverflow.com/questions/4020539/process-escape-sequences-in-a-string-in-python _HEXCHAR = '[a-fA-F0-9]' @@ -36,12 +38,14 @@ _ESCAPE_SEQUENCE_RE = re.compile(r''' | \\[\\'"abfnrtv] # Single-character escapes )'''.format(_HEXCHAR*8, _HEXCHAR*4, _HEXCHAR*2), re.UNICODE | re.VERBOSE) + def _decode_escapes(s): def decode_match(match): return codecs.decode(match.group(0), 'unicode-escape') return _ESCAPE_SEQUENCE_RE.sub(decode_match, s) + def parse_kv(args, check_raw=False): ''' Convert a string of key/value items to a dict. If any free-form params @@ -50,9 +54,7 @@ def parse_kv(args, check_raw=False): they will simply be ignored. ''' - ### FIXME: args should already be a unicode string - from ansible.utils.unicode import to_unicode - args = to_unicode(args, nonstring='passthru') + args = to_text(args, nonstring='passthru') options = {} if args is not None: @@ -60,7 +62,7 @@ def parse_kv(args, check_raw=False): vargs = split_args(args) except ValueError as ve: if 'no closing quotation' in str(ve).lower(): - raise AnsibleParsingError("error parsing argument string, try quoting the entire line.") + raise AnsibleParserError("error parsing argument string, try quoting the entire line.") else: raise @@ -99,6 +101,7 @@ def parse_kv(args, check_raw=False): return options + def _get_quote_state(token, quote_char): ''' the goal of this block is to determine if the quoted string @@ -118,6 +121,7 @@ def _get_quote_state(token, quote_char): quote_char = cur_char return quote_char + def _count_jinja2_blocks(token, cur_depth, open_token, close_token): ''' this function counts the number of opening/closing blocks for a @@ -132,6 +136,7 @@ def _count_jinja2_blocks(token, cur_depth, open_token, close_token): cur_depth = 0 return cur_depth + def split_args(args): ''' Splits args on whitespace, but intelligently reassembles @@ -166,9 +171,9 @@ def split_args(args): quote_char = None inside_quotes = False - print_depth = 0 # used to count nested jinja2 {{ }} blocks - block_depth = 0 # used to count nested jinja2 {% %} blocks - comment_depth = 0 # used to count nested jinja2 {# #} blocks + print_depth = 0 # used to count nested jinja2 {{ }} blocks + block_depth = 0 # used to count nested jinja2 {% %} blocks + comment_depth = 0 # used to count nested jinja2 {# #} blocks # now we loop over each split chunk, coalescing tokens if the white space # split occurred within quotes or a jinja2 block of some kind diff --git a/lib/ansible/parsing/vault/__init__.py b/lib/ansible/parsing/vault/__init__.py index 42cebeb551..1845210d3d 100644 --- a/lib/ansible/parsing/vault/__init__.py +++ b/lib/ansible/parsing/vault/__init__.py @@ -29,16 +29,10 @@ from ansible.errors import AnsibleError from hashlib import sha256 from binascii import hexlify from binascii import unhexlify - -try: - from __main__ import display -except ImportError: - from ansible.utils.display import Display - display = Display() +from hashlib import md5 # Note: Only used for loading obsolete VaultAES files. All files are written # using the newer VaultAES256 which does not require md5 -from hashlib import md5 try: from Crypto.Hash import SHA256, HMAC @@ -67,6 +61,15 @@ try: except ImportError: HAS_AES = False +from ansible.compat.six import PY3 +from ansible.module_utils._text import to_bytes, to_text + +try: + from __main__ import display +except ImportError: + from ansible.utils.display import Display + display = Display() + # OpenSSL pbkdf2_hmac HAS_PBKDF2HMAC = False try: @@ -81,12 +84,11 @@ except Exception as e: import traceback display.debug("Traceback from import of cryptography was {0}".format(traceback.format_exc())) -from ansible.compat.six import PY3 -from ansible.utils.unicode import to_unicode, to_bytes HAS_ANY_PBKDF2HMAC = HAS_PBKDF2 or HAS_PBKDF2HMAC -CRYPTO_UPGRADE = "ansible-vault requires a newer version of pycrypto than the one installed on your platform. You may fix this with OS-specific commands such as: yum install python-devel; rpm -e --nodeps python-crypto; pip install pycrypto" +CRYPTO_UPGRADE = "ansible-vault requires a newer version of pycrypto than the one installed on your platform." \ + " You may fix this with OS-specific commands such as: yum install python-devel; rpm -e --nodeps python-crypto; pip install pycrypto" b_HEADER = b'$ANSIBLE_VAULT' HEADER = '$ANSIBLE_VAULT' @@ -105,6 +107,7 @@ def check_prereqs(): class AnsibleVaultError(AnsibleError): pass + def is_encrypted(b_data): """ Test if this is vault encrypted data blob @@ -116,6 +119,7 @@ def is_encrypted(b_data): return True return False + def is_encrypted_file(file_obj): """Test if the contents of a file obj are a vault encrypted data blob. @@ -252,7 +256,7 @@ class VaultLib: b_header = HEADER.encode('utf-8') header = b';'.join([b_header, self.b_version, - to_bytes(self.cipher_name,'utf-8',errors='strict')]) + to_bytes(self.cipher_name,'utf-8', errors='strict')]) tmpdata = [header] tmpdata += [b_data[i:i + 80] for i in range(0, len(b_data), 80)] tmpdata += [b''] @@ -278,7 +282,7 @@ class VaultLib: tmpheader = tmpdata[0].strip().split(b';') self.b_version = tmpheader[1].strip() - self.cipher_name = to_unicode(tmpheader[2].strip()) + self.cipher_name = to_text(tmpheader[2].strip()) clean_data = b''.join(tmpdata[1:]) return clean_data @@ -306,7 +310,7 @@ class VaultEditor: file_len = os.path.getsize(tmp_path) - if file_len > 0: # avoid work when file was empty + if file_len > 0: # avoid work when file was empty max_chunk_len = min(1024*1024*2, file_len) passes = 3 @@ -321,7 +325,7 @@ class VaultEditor: fh.write(data) fh.write(data[:file_len % chunk_len]) - assert(fh.tell() == file_len) # FIXME remove this assert once we have unittests to check its accuracy + assert(fh.tell() == file_len) # FIXME remove this assert once we have unittests to check its accuracy os.fsync(fh) def _shred_file(self, tmp_path): @@ -528,6 +532,7 @@ class VaultEditor: return editor + class VaultFile(object): def __init__(self, password, filename): @@ -568,6 +573,7 @@ class VaultFile(object): else: return self.filename + ######################################## # CIPHERS # ######################################## diff --git a/lib/ansible/parsing/yaml/constructor.py b/lib/ansible/parsing/yaml/constructor.py index a67481a781..0a53fb1c7f 100644 --- a/lib/ansible/parsing/yaml/constructor.py +++ b/lib/ansible/parsing/yaml/constructor.py @@ -22,12 +22,11 @@ __metaclass__ = type from yaml.constructor import Constructor, ConstructorError from yaml.nodes import MappingNode +from ansible.module_utils._text import to_bytes +from ansible.parsing.vault import VaultLib from ansible.parsing.yaml.objects import AnsibleMapping, AnsibleSequence, AnsibleUnicode from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode - from ansible.vars.unsafe_proxy import wrap_var -from ansible.parsing.vault import VaultLib -from ansible.utils.unicode import to_bytes try: from __main__ import display @@ -74,7 +73,8 @@ class AnsibleConstructor(Constructor): "found unacceptable key (%s)" % exc, key_node.start_mark) if key in mapping: - display.warning(u'While constructing a mapping from {1}, line {2}, column {3}, found a duplicate dict key ({0}). Using last defined value only.'.format(key, *mapping.ansible_pos)) + display.warning(u'While constructing a mapping from {1}, line {2}, column {3}, found a duplicate dict key ({0}).' + u' Using last defined value only.'.format(key, *mapping.ansible_pos)) value = self.construct_object(value_node, deep=deep) mapping[key] = value diff --git a/lib/ansible/parsing/yaml/objects.py b/lib/ansible/parsing/yaml/objects.py index 60553d5cb4..ea0b274c3d 100644 --- a/lib/ansible/parsing/yaml/objects.py +++ b/lib/ansible/parsing/yaml/objects.py @@ -22,8 +22,7 @@ __metaclass__ = type import yaml from ansible.compat.six import text_type -from ansible.errors import AnsibleError -from ansible.utils.unicode import to_bytes +from ansible.module_utils._text import to_bytes class AnsibleBaseYAMLObject(object): diff --git a/lib/ansible/playbook/base.py b/lib/ansible/playbook/base.py index 60cda79070..77fd6890b5 100644 --- a/lib/ansible/playbook/base.py +++ b/lib/ansible/playbook/base.py @@ -19,25 +19,22 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type -import collections import itertools import operator import uuid -from copy import copy as shallowcopy, deepcopy +from copy import copy as shallowcopy from functools import partial -from inspect import getmembers - -from ansible.compat.six import iteritems, string_types, with_metaclass from jinja2.exceptions import UndefinedError +from ansible.compat.six import iteritems, string_types, with_metaclass from ansible.errors import AnsibleParserError, AnsibleUndefinedVariable -from ansible.parsing.dataloader import DataLoader +from ansible.module_utils._text import to_text from ansible.playbook.attribute import Attribute, FieldAttribute +from ansible.parsing.dataloader import DataLoader from ansible.utils.boolean import boolean from ansible.utils.vars import combine_vars, isidentifier -from ansible.utils.unicode import to_unicode try: from __main__ import display @@ -52,6 +49,7 @@ def _generic_g(prop_name, self): except KeyError: raise AttributeError("'%s' object has no attribute '%s'" % (self.__class__.__name__, prop_name)) + def _generic_g_method(prop_name, self): try: if self._squashed: @@ -61,6 +59,7 @@ def _generic_g_method(prop_name, self): except KeyError: raise AttributeError("'%s' object has no attribute '%s'" % (self.__class__.__name__, prop_name)) + def _generic_g_parent(prop_name, self): try: value = self._attributes[prop_name] @@ -74,12 +73,15 @@ def _generic_g_parent(prop_name, self): return value + def _generic_s(prop_name, self, value): self._attributes[prop_name] = value + def _generic_d(prop_name, self): del self._attributes[prop_name] + class BaseMeta(type): """ @@ -142,6 +144,7 @@ class BaseMeta(type): return super(BaseMeta, cls).__new__(cls, name, parents, dct) + class Base(with_metaclass(BaseMeta, object)): # connection/transport @@ -376,7 +379,7 @@ class Base(with_metaclass(BaseMeta, object)): # and make sure the attribute is of the type it should be if value is not None: if attribute.isa == 'string': - value = to_unicode(value) + value = to_text(value) elif attribute.isa == 'int': value = int(value) elif attribute.isa == 'float': @@ -395,8 +398,8 @@ class Base(with_metaclass(BaseMeta, object)): elif not isinstance(value, list): if isinstance(value, string_types) and attribute.isa == 'barelist': display.deprecated( - "Using comma separated values for a list has been deprecated. " \ - "You should instead use the correct YAML syntax for lists. " \ + "Using comma separated values for a list has been deprecated. " + "You should instead use the correct YAML syntax for lists. " ) value = value.split(',') else: @@ -532,4 +535,3 @@ class Base(with_metaclass(BaseMeta, object)): setattr(self, '_uuid', data.get('uuid')) self._finalized = data.get('finalized', False) self._squashed = data.get('squashed', False) - diff --git a/lib/ansible/playbook/task.py b/lib/ansible/playbook/task.py index f5bd653314..e987045438 100644 --- a/lib/ansible/playbook/task.py +++ b/lib/ansible/playbook/task.py @@ -22,12 +22,10 @@ __metaclass__ = type import os from ansible.compat.six import iteritems, string_types - from ansible.errors import AnsibleError, AnsibleParserError - +from ansible.module_utils._text import to_native from ansible.parsing.mod_args import ModuleArgsParser from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleMapping, AnsibleUnicode - from ansible.plugins import lookup_loader from ansible.playbook.attribute import FieldAttribute from ansible.playbook.base import Base @@ -38,7 +36,6 @@ from ansible.playbook.loop_control import LoopControl from ansible.playbook.role import Role from ansible.playbook.taggable import Taggable -from ansible.utils.unicode import to_str try: from __main__ import display @@ -182,7 +179,7 @@ class Task(Base, Conditional, Taggable, Become): try: (action, args, delegate_to) = args_parser.parse() except AnsibleParserError as e: - raise AnsibleParserError(to_str(e), obj=ds) + raise AnsibleParserError(to_native(e), obj=ds) # the command/shell/script modules used to support the `cmd` arg, # which corresponds to what we now call _raw_params, so move that @@ -232,11 +229,11 @@ class Task(Base, Conditional, Taggable, Become): def _load_loop_control(self, attr, ds): if not isinstance(ds, dict): - raise AnsibleParserError( - "the `loop_control` value must be specified as a dictionary and cannot " \ - "be a variable itself (though it can contain variables)", - obj=ds, - ) + raise AnsibleParserError( + "the `loop_control` value must be specified as a dictionary and cannot " + "be a variable itself (though it can contain variables)", + obj=ds, + ) return LoopControl.load(data=ds, variable_manager=self._variable_manager, loader=self._loader) @@ -267,18 +264,18 @@ class Task(Base, Conditional, Taggable, Become): return dict() elif isinstance(value, list): - if len(value) == 1: + if len(value) == 1: return templar.template(value[0], convert_bare=True) else: env = [] for env_item in value: if isinstance(env_item, (string_types, AnsibleUnicode)) and env_item in templar._available_variables.keys(): - env[env_item] = templar.template(env_item, convert_bare=True) + env[env_item] = templar.template(env_item, convert_bare=True) elif isinstance(value, dict): env = dict() for env_item in value: if isinstance(env_item, (string_types, AnsibleUnicode)) and env_item in templar._available_variables.keys(): - env[env_item] = templar.template(value[env_item], convert_bare=True) + env[env_item] = templar.template(value[env_item], convert_bare=True) # at this point it should be a simple string return templar.template(value, convert_bare=True) @@ -436,7 +433,7 @@ class Task(Base, Conditional, Taggable, Become): ''' path_stack = [] - dep_chain = self.get_dep_chain() + dep_chain = self.get_dep_chain() # inside role: add the dependency chain from current to dependant if dep_chain: path_stack.extend(reversed([x._role_path for x in dep_chain])) @@ -452,4 +449,3 @@ class Task(Base, Conditional, Taggable, Become): if self._parent: return self._parent.all_parents_static() return True - diff --git a/lib/ansible/plugins/__init__.py b/lib/ansible/plugins/__init__.py index 0a2a0b1474..f5a6f112a8 100644 --- a/lib/ansible/plugins/__init__.py +++ b/lib/ansible/plugins/__init__.py @@ -31,7 +31,8 @@ import warnings from collections import defaultdict from ansible import constants as C -from ansible.utils.unicode import to_unicode +from ansible.module_utils._text import to_text + try: from __main__ import display @@ -44,9 +45,11 @@ MODULE_CACHE = {} PATH_CACHE = {} PLUGIN_PATH_CACHE = {} + def get_all_plugin_loaders(): return [(name, obj) for (name, obj) in inspect.getmembers(sys.modules[__name__]) if isinstance(obj, PluginLoader)] + class PluginLoader: ''' @@ -72,11 +75,11 @@ class PluginLoader: self.config = config - if not class_name in MODULE_CACHE: + if class_name not in MODULE_CACHE: MODULE_CACHE[class_name] = {} - if not class_name in PATH_CACHE: + if class_name not in PATH_CACHE: PATH_CACHE[class_name] = None - if not class_name in PLUGIN_PATH_CACHE: + if class_name not in PLUGIN_PATH_CACHE: PLUGIN_PATH_CACHE[class_name] = defaultdict(dict) self._module_cache = MODULE_CACHE[class_name] @@ -140,9 +143,9 @@ class PluginLoader: results = [] results.append(dir) for root, subdirs, files in os.walk(dir, followlinks=True): - if '__init__.py' in files: - for x in subdirs: - results.append(os.path.join(root,x)) + if '__init__.py' in files: + for x in subdirs: + results.append(os.path.join(root,x)) return results def _get_package_paths(self): @@ -250,7 +253,7 @@ class PluginLoader: try: full_paths = (os.path.join(path, f) for f in os.listdir(path)) except OSError as e: - display.warning("Error accessing plugin paths: %s" % to_unicode(e)) + display.warning("Error accessing plugin paths: %s" % to_text(e)) for full_path in (f for f in full_paths if os.path.isfile(f) and not f.endswith('__init__.py')): full_name = os.path.basename(full_path) @@ -358,7 +361,7 @@ class PluginLoader: def _display_plugin_load(self, class_name, name, searched_paths, path, found_in_cache=None, class_only=None): msg = 'Loading %s \'%s\' from %s' % (class_name, os.path.basename(name), path) - + if len(searched_paths) > 1: msg = '%s (searched paths: %s)' % (msg, self.format_paths(searched_paths)) @@ -389,7 +392,7 @@ class PluginLoader: try: obj = getattr(self._module_cache[path], self.class_name) except AttributeError as e: - display.warning("Skipping plugin (%s) as it seems to be invalid: %s" % (path, to_unicode(e))) + display.warning("Skipping plugin (%s) as it seems to be invalid: %s" % (path, to_text(e))) continue if self.base_class: @@ -398,11 +401,11 @@ class PluginLoader: module = __import__(self.package, fromlist=[self.base_class]) # Check whether this obj has the required base class. try: - plugin_class = getattr(module, self.base_class) + plugin_class = getattr(module, self.base_class) except AttributeError: - continue + continue if not issubclass(obj, plugin_class): - continue + continue self._display_plugin_load(self.class_name, name, self._searched_paths, path, found_in_cache=found_in_cache, class_only=class_only) diff --git a/lib/ansible/plugins/action/__init__.py b/lib/ansible/plugins/action/__init__.py index ef9920033d..efe0e869cb 100644 --- a/lib/ansible/plugins/action/__init__.py +++ b/lib/ansible/plugins/action/__init__.py @@ -35,9 +35,10 @@ from ansible.compat.six import binary_type, text_type, iteritems, with_metaclass from ansible import constants as C from ansible.errors import AnsibleError, AnsibleConnectionFailure from ansible.executor.module_common import modify_module -from ansible.release import __version__ +from ansible.module_utils._text import to_bytes, to_native, to_text from ansible.parsing.utils.jsonify import jsonify -from ansible.utils.unicode import to_bytes, to_str, to_unicode +from ansible.release import __version__ + try: from __main__ import display @@ -86,7 +87,7 @@ class ActionBase(with_metaclass(ABCMeta, object)): * Module parameters. These are stored in self._task.args """ # store the module invocation details into the results - results = {} + results = {} if self._task.async == 0: results['invocation'] = dict( module_name = self._task.action, @@ -146,7 +147,8 @@ class ActionBase(with_metaclass(ABCMeta, object)): "run 'git submodule update --init --recursive' to correct this problem." % (module_name)) # insert shared code and arguments into the module - (module_data, module_style, module_shebang) = modify_module(module_name, module_path, module_args, task_vars=task_vars, module_compression=self._play_context.module_compression) + (module_data, module_style, module_shebang) = modify_module(module_name, module_path, module_args, + task_vars=task_vars, module_compression=self._play_context.module_compression) return (module_style, module_shebang, module_data, module_path) @@ -283,10 +285,10 @@ class ActionBase(with_metaclass(ABCMeta, object)): afd, afile = tempfile.mkstemp() afo = os.fdopen(afd, 'wb') try: - data = to_bytes(data, errors='strict') + data = to_bytes(data, errors='surrogate_or_strict') afo.write(data) except Exception as e: - raise AnsibleError("failure writing module data to temporary file for transfer: %s" % str(e)) + raise AnsibleError("failure writing module data to temporary file for transfer: %s" % to_native(e)) afo.flush() afo.close() @@ -372,17 +374,22 @@ class ActionBase(with_metaclass(ABCMeta, object)): res = self._remote_chown(remote_paths, self._play_context.become_user) if res['rc'] != 0 and remote_user == 'root': # chown failed even if remove_user is root - raise AnsibleError('Failed to change ownership of the temporary files Ansible needs to create despite connecting as root. Unprivileged become user would be unable to read the file.') + raise AnsibleError('Failed to change ownership of the temporary files Ansible needs to create despite connecting as root.' + ' Unprivileged become user would be unable to read the file.') elif res['rc'] != 0: if C.ALLOW_WORLD_READABLE_TMPFILES: # chown and fs acls failed -- do things this insecure # way only if the user opted in in the config file - display.warning('Using world-readable permissions for temporary files Ansible needs to create when becoming an unprivileged user which may be insecure. For information on securing this, see https://docs.ansible.com/ansible/become.html#becoming-an-unprivileged-user') + display.warning('Using world-readable permissions for temporary files Ansible needs to create when becoming an unprivileged user.' + ' This may be insecure. For information on securing this, see' + ' https://docs.ansible.com/ansible/become.html#becoming-an-unprivileged-user') res = self._remote_chmod(remote_paths, 'a+%s' % mode) if res['rc'] != 0: raise AnsibleError('Failed to set file mode on remote files (rc: {0}, err: {1})'.format(res['rc'], res['stderr'])) else: - raise AnsibleError('Failed to set permissions on the temporary files Ansible needs to create when becoming an unprivileged user (rc: {0}, err: {1}). For information on working around this, see https://docs.ansible.com/ansible/become.html#becoming-an-unprivileged-user'.format(res['rc'], res['stderr'])) + raise AnsibleError('Failed to set permissions on the temporary files Ansible needs to create when becoming an unprivileged user' + ' (rc: {0}, err: {1}). For information on working around this,' + ' see https://docs.ansible.com/ansible/become.html#becoming-an-unprivileged-user'.format(res['rc'], res['stderr'])) elif execute: # Can't depend on the file being transferred with execute # permissions. Only need user perms because no become was @@ -438,7 +445,7 @@ class ActionBase(with_metaclass(ABCMeta, object)): mystat['stat']['checksum'] = '1' # happens sometimes when it is a dir and not on bsd - if not 'checksum' in mystat['stat']: + if 'checksum' not in mystat['stat']: mystat['stat']['checksum'] = '' return mystat['stat'] @@ -453,26 +460,25 @@ class ActionBase(with_metaclass(ABCMeta, object)): 3 = its a directory, not a file 4 = stat module failed, likely due to not finding python ''' - x = "0" # unknown error has occured + x = "0" # unknown error has occured try: remote_stat = self._execute_remote_stat(path, all_vars, follow=follow) if remote_stat['exists'] and remote_stat['isdir']: - x = "3" # its a directory not a file + x = "3" # its a directory not a file else: - x = remote_stat['checksum'] # if 1, file is missing + x = remote_stat['checksum'] # if 1, file is missing except AnsibleError as e: - errormsg = to_unicode(e) - if errormsg.endswith('Permission denied'): - x = "2" # cannot read file - elif errormsg.endswith('MODULE FAILURE'): - x = "4" # python not found or module uncaught exception + errormsg = to_text(e) + if errormsg.endswith(u'Permission denied'): + x = "2" # cannot read file + elif errormsg.endswith(u'MODULE FAILURE'): + x = "4" # python not found or module uncaught exception finally: return x - def _remote_expand_user(self, path): ''' takes a remote path and performs tilde expansion on the remote host ''' - if not path.startswith('~'): # FIXME: Windows paths may start with "~ instead of just ~ + if not path.startswith('~'): # FIXME: Windows paths may start with "~ instead of just ~ return path # FIXME: Can't use os.path.sep for Windows paths. @@ -681,7 +687,8 @@ class ActionBase(with_metaclass(ABCMeta, object)): tmp_rm_res = self._low_level_execute_command(tmp_rm_cmd, sudoable=False) tmp_rm_data = self._parse_returned_data(tmp_rm_res) if tmp_rm_data.get('rc', 0) != 0: - display.warning('Error deleting remote temporary files (rc: {0}, stderr: {1})'.format(tmp_rm_res.get('rc'), tmp_rm_res.get('stderr', 'No error string available.'))) + display.warning('Error deleting remote temporary files (rc: {0}, stderr: {1})'.format(tmp_rm_res.get('rc'), + tmp_rm_res.get('stderr', 'No error string available.'))) # parse the main result data = self._parse_returned_data(res) @@ -709,7 +716,7 @@ class ActionBase(with_metaclass(ABCMeta, object)): data['exception'] = res['stderr'] return data - def _low_level_execute_command(self, cmd, sudoable=True, in_data=None, executable=None, encoding_errors='replace'): + def _low_level_execute_command(self, cmd, sudoable=True, in_data=None, executable=None, encoding_errors='surrogate_or_replace'): ''' This is the function which executes the low level shell command, which may be commands to create/remove directories for temporary files, or to @@ -758,16 +765,16 @@ class ActionBase(with_metaclass(ABCMeta, object)): # stdout and stderr may be either a file-like or a bytes object. # Convert either one to a text type if isinstance(stdout, binary_type): - out = to_unicode(stdout, errors=encoding_errors) + out = to_text(stdout, errors=encoding_errors) elif not isinstance(stdout, text_type): - out = to_unicode(b''.join(stdout.readlines()), errors=encoding_errors) + out = to_text(b''.join(stdout.readlines()), errors=encoding_errors) else: out = stdout if isinstance(stderr, binary_type): - err = to_unicode(stderr, errors=encoding_errors) + err = to_text(stderr, errors=encoding_errors) elif not isinstance(stderr, text_type): - err = to_unicode(b''.join(stderr.readlines()), errors=encoding_errors) + err = to_text(b''.join(stderr.readlines()), errors=encoding_errors) else: err = stderr @@ -871,7 +878,6 @@ class ActionBase(with_metaclass(ABCMeta, object)): result = self._loader.path_dwim_relative_stack(path_stack, dirname, needle) if result is None: - raise AnsibleError("Unable to find '%s' in expected paths." % to_str(needle)) + raise AnsibleError("Unable to find '%s' in expected paths." % to_native(needle)) return result - diff --git a/lib/ansible/plugins/action/assemble.py b/lib/ansible/plugins/action/assemble.py index 4d1f7a6cc0..f02b9bfa25 100644 --- a/lib/ansible/plugins/action/assemble.py +++ b/lib/ansible/plugins/action/assemble.py @@ -24,10 +24,10 @@ import tempfile import re from ansible.errors import AnsibleError +from ansible.module_utils._text import to_native, to_text from ansible.plugins.action import ActionBase from ansible.utils.boolean import boolean from ansible.utils.hashing import checksum_s -from ansible.utils.unicode import to_str, to_unicode class ActionModule(ActionBase): @@ -42,7 +42,7 @@ class ActionModule(ActionBase): delimit_me = False add_newline = False - for f in (to_unicode(p, errors='strict') for p in sorted(os.listdir(src_path))): + for f in (to_text(p, errors='surrogate_or_strict') for p in sorted(os.listdir(src_path))): if compiled_regexp and not compiled_regexp.search(f): continue fragment = u"%s/%s" % (src_path, f) @@ -114,7 +114,7 @@ class ActionModule(ActionBase): src = self._find_needle('files', src) except AnsibleError as e: result['failed'] = True - result['msg'] = to_str(e) + result['msg'] = to_native(e) return result if not os.path.isdir(src): diff --git a/lib/ansible/plugins/action/async.py b/lib/ansible/plugins/action/async.py index 5e1bc467d2..6c348c7d9f 100644 --- a/lib/ansible/plugins/action/async.py +++ b/lib/ansible/plugins/action/async.py @@ -22,9 +22,10 @@ import pipes import random from ansible import constants as C -from ansible.plugins.action import ActionBase from ansible.compat.six import iteritems -from ansible.utils.unicode import to_unicode +from ansible.module_utils._text import to_text +from ansible.plugins.action import ActionBase + class ActionModule(ActionBase): @@ -76,7 +77,7 @@ class ActionModule(ActionBase): elif module_style == 'old': args_data = "" for k, v in iteritems(module_args): - args_data += '%s="%s" ' % (k, pipes.quote(to_unicode(v))) + args_data += '%s="%s" ' % (k, pipes.quote(to_text(v))) argsfile = self._transfer_data(self._connection._shell.join_path(tmp, 'arguments'), args_data) remote_paths = tmp, remote_module_path, remote_async_module_path @@ -100,7 +101,7 @@ class ActionModule(ActionBase): if not self._should_remove_tmp_path(tmp): async_cmd.append("-preserve_tmp") - async_cmd = " ".join([to_unicode(x) for x in async_cmd]) + async_cmd = " ".join(to_text(x) for x in async_cmd) result.update(self._low_level_execute_command(cmd=async_cmd)) result['changed'] = True diff --git a/lib/ansible/plugins/action/copy.py b/lib/ansible/plugins/action/copy.py index 140c714c27..f71ecadc77 100644 --- a/lib/ansible/plugins/action/copy.py +++ b/lib/ansible/plugins/action/copy.py @@ -24,10 +24,10 @@ import os import tempfile from ansible.errors import AnsibleError +from ansible.module_utils._text import to_bytes, to_native, to_text from ansible.plugins.action import ActionBase from ansible.utils.boolean import boolean from ansible.utils.hashing import checksum -from ansible.utils.unicode import to_bytes, to_str, to_unicode class ActionModule(ActionBase): @@ -81,7 +81,7 @@ class ActionModule(ActionBase): source = content_tempfile except Exception as err: result['failed'] = True - result['msg'] = "could not write content temp file: %s" % to_str(err) + result['msg'] = "could not write content temp file: %s" % to_native(err) return result # if we have first_available_file in our vars @@ -91,19 +91,19 @@ class ActionModule(ActionBase): elif remote_src: result.update(self._execute_module(module_name='copy', module_args=self._task.args, task_vars=task_vars, delete_remote_tmp=False)) return result - else: # find in expected paths + else: # find in expected paths try: source = self._find_needle('files', source) except AnsibleError as e: result['failed'] = True - result['msg'] = to_unicode(e) + result['msg'] = to_text(e) return result # A list of source file tuples (full_path, relative_path) which will try to copy to the destination source_files = [] # If source is a directory populate our list else source is a file and translate it to a tuple. - if os.path.isdir(to_bytes(source, errors='strict')): + if os.path.isdir(to_bytes(source, errors='surrogate_or_strict')): # Get the amount of spaces to remove to get the relative path. if source_trailing_slash: sz = len(source) @@ -113,7 +113,7 @@ class ActionModule(ActionBase): # Walk the directory and append the file tuples to source_files. for base_path, sub_folders, files in os.walk(to_bytes(source)): for file in files: - full_path = to_unicode(os.path.join(base_path, file), errors='strict') + full_path = to_text(os.path.join(base_path, file), errors='surrogate_or_strict') rel_path = full_path[sz:] if rel_path.startswith('/'): rel_path = rel_path[1:] @@ -247,7 +247,9 @@ class ActionModule(ActionBase): if 'content' in new_module_args: del new_module_args['content'] - module_return = self._execute_module(module_name='copy', module_args=new_module_args, task_vars=task_vars, tmp=tmp, delete_remote_tmp=delete_remote_tmp) + module_return = self._execute_module(module_name='copy', + module_args=new_module_args, task_vars=task_vars, + tmp=tmp, delete_remote_tmp=delete_remote_tmp) module_executed = True else: @@ -272,7 +274,9 @@ class ActionModule(ActionBase): ) # Execute the file module. - module_return = self._execute_module(module_name='file', module_args=new_module_args, task_vars=task_vars, tmp=tmp, delete_remote_tmp=delete_remote_tmp) + module_return = self._execute_module(module_name='file', + module_args=new_module_args, task_vars=task_vars, + tmp=tmp, delete_remote_tmp=delete_remote_tmp) module_executed = True if not module_return.get('checksum'): diff --git a/lib/ansible/plugins/action/debug.py b/lib/ansible/plugins/action/debug.py index aa2bee4768..f9f87adef5 100644 --- a/lib/ansible/plugins/action/debug.py +++ b/lib/ansible/plugins/action/debug.py @@ -20,8 +20,8 @@ __metaclass__ = type from ansible.compat.six import string_types from ansible.errors import AnsibleUndefinedVariable +from ansible.module_utils._text import to_text from ansible.plugins.action import ActionBase -from ansible.utils.unicode import to_unicode class ActionModule(ActionBase): @@ -66,7 +66,7 @@ class ActionModule(ActionBase): if isinstance(self._task.args['var'], (list, dict)): # If var is a list or dict, use the type as key to display - result[to_unicode(type(self._task.args['var']))] = results + result[to_text(type(self._task.args['var']))] = results else: result[self._task.args['var']] = results else: diff --git a/lib/ansible/plugins/action/fetch.py b/lib/ansible/plugins/action/fetch.py index 693c8ddc8d..69cec30742 100644 --- a/lib/ansible/plugins/action/fetch.py +++ b/lib/ansible/plugins/action/fetch.py @@ -21,11 +21,11 @@ import os import base64 from ansible.errors import AnsibleError +from ansible.module_utils._text import to_bytes from ansible.plugins.action import ActionBase from ansible.utils.boolean import boolean from ansible.utils.hashing import checksum, checksum_s, md5, secure_hash from ansible.utils.path import makedirs_safe -from ansible.utils.unicode import to_bytes class ActionModule(ActionBase): @@ -160,7 +160,7 @@ class ActionModule(ActionBase): self._connection.fetch_file(source, dest) else: try: - f = open(to_bytes(dest, errors='strict'), 'w') + f = open(to_bytes(dest, errors='surrogate_or_strict'), 'wb') f.write(remote_data) f.close() except (IOError, OSError) as e: diff --git a/lib/ansible/plugins/action/include_vars.py b/lib/ansible/plugins/action/include_vars.py index b1867d48f5..97c9a8c68d 100644 --- a/lib/ansible/plugins/action/include_vars.py +++ b/lib/ansible/plugins/action/include_vars.py @@ -22,8 +22,8 @@ from os import path, walk import re from ansible.errors import AnsibleError +from ansible.module_utils._text import to_native from ansible.plugins.action import ActionBase -from ansible.utils.unicode import to_str class ActionModule(ActionBase): @@ -137,7 +137,7 @@ class ActionModule(ActionBase): results.update(updated_results) except AnsibleError as e: - err_msg = to_str(e) + err_msg = to_native(e) raise AnsibleError(err_msg) if self.return_results_as_name: diff --git a/lib/ansible/plugins/action/net_config.py b/lib/ansible/plugins/action/net_config.py index 216320bfc4..5cb3e646b9 100644 --- a/lib/ansible/plugins/action/net_config.py +++ b/lib/ansible/plugins/action/net_config.py @@ -26,10 +26,12 @@ import glob import urlparse from ansible.plugins.action import ActionBase -from ansible.utils.unicode import to_unicode +from ansible.module_utils._text import to_text + PRIVATE_KEYS_RE = re.compile('__.+__') + class ActionModule(ActionBase): TRANSFERS_FILES = False @@ -56,7 +58,6 @@ class ActionModule(ActionBase): result['__backup__']) result['backup_path'] = filepath - # strip out any keys that have two leading and two trailing # underscore characters for key in result.keys(): @@ -98,7 +99,7 @@ class ActionModule(ActionBase): try: with open(source, 'r') as f: - template_data = to_unicode(f.read()) + template_data = to_text(f.read()) except IOError: return dict(failed=True, msg='unable to load src file') @@ -114,5 +115,3 @@ class ActionModule(ActionBase): searchpath.append(os.path.dirname(source)) self._templar.environment.loader.searchpath = searchpath self._task.args['src'] = self._templar.template(template_data) - - diff --git a/lib/ansible/plugins/action/net_template.py b/lib/ansible/plugins/action/net_template.py index 1561e7e215..f76506aaae 100644 --- a/lib/ansible/plugins/action/net_template.py +++ b/lib/ansible/plugins/action/net_template.py @@ -19,18 +19,18 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type -import sys import os import time import glob import urlparse +from ansible.module_utils._text import to_text from ansible.plugins.action import ActionBase -from ansible.utils.boolean import boolean -from ansible.utils.unicode import to_unicode + BOOLEANS = ('true', 'false', 'yes', 'no') + class ActionModule(ActionBase): TRANSFERS_FILES = False @@ -92,7 +92,7 @@ class ActionModule(ActionBase): try: with open(source, 'r') as f: - template_data = to_unicode(f.read()) + template_data = to_text(f.read()) except IOError: return dict(failed=True, msg='unable to load src file') @@ -108,5 +108,3 @@ class ActionModule(ActionBase): searchpath.append(os.path.dirname(source)) self._templar.environment.loader.searchpath = searchpath self._task.args['src'] = self._templar.template(template_data) - - diff --git a/lib/ansible/plugins/action/patch.py b/lib/ansible/plugins/action/patch.py index db1a2eaba7..94bcf67746 100644 --- a/lib/ansible/plugins/action/patch.py +++ b/lib/ansible/plugins/action/patch.py @@ -20,10 +20,10 @@ __metaclass__ = type import os +from ansible.errors import AnsibleError +from ansible.module_utils._text import to_native from ansible.plugins.action import ActionBase from ansible.utils.boolean import boolean -from ansible.errors import AnsibleError -from ansible.utils.unicode import to_str class ActionModule(ActionBase): @@ -52,7 +52,7 @@ class ActionModule(ActionBase): src = self._find_needle('files', src) except AnsibleError as e: result['failed'] = True - result['msg'] = to_str(e) + result['msg'] = to_native(e) return result # create the remote tmp dir if needed, and put the source file there diff --git a/lib/ansible/plugins/action/script.py b/lib/ansible/plugins/action/script.py index 286142fafc..44cbff02e1 100644 --- a/lib/ansible/plugins/action/script.py +++ b/lib/ansible/plugins/action/script.py @@ -19,15 +19,14 @@ __metaclass__ = type import os -from ansible.plugins.action import ActionBase from ansible.errors import AnsibleError -from ansible.utils.unicode import to_str +from ansible.module_utils._text import to_native +from ansible.plugins.action import ActionBase class ActionModule(ActionBase): TRANSFERS_FILES = True - def run(self, tmp=None, task_vars=None): ''' handler for file transfer operations ''' if task_vars is None: @@ -74,7 +73,7 @@ class ActionModule(ActionBase): try: source = self._loader.get_real_file(self._find_needle('files', source)) except AnsibleError as e: - return dict(failed=True, msg=to_str(e)) + return dict(failed=True, msg=to_native(e)) # transfer the file to a remote tmp location tmp_src = self._connection._shell.join_path(tmp, os.path.basename(source)) diff --git a/lib/ansible/plugins/action/template.py b/lib/ansible/plugins/action/template.py index 895ac835c8..1daca6073e 100644 --- a/lib/ansible/plugins/action/template.py +++ b/lib/ansible/plugins/action/template.py @@ -23,12 +23,11 @@ import pwd import time from ansible import constants as C +from ansible.errors import AnsibleError +from ansible.module_utils._text import to_bytes, to_native, to_text from ansible.plugins.action import ActionBase from ansible.utils.hashing import checksum_s from ansible.utils.boolean import boolean -from ansible.utils.unicode import to_bytes, to_unicode, to_str -from ansible.errors import AnsibleError - class ActionModule(ActionBase): @@ -78,7 +77,7 @@ class ActionModule(ActionBase): source = self._find_needle('templates', source) except AnsibleError as e: result['failed'] = True - result['msg'] = to_str(e) + result['msg'] = to_native(e) if 'failed' in result: return result @@ -96,7 +95,7 @@ class ActionModule(ActionBase): b_source = to_bytes(source) try: with open(b_source, 'r') as f: - template_data = to_unicode(f.read()) + template_data = to_text(f.read()) try: template_uid = pwd.getpwuid(os.stat(b_source).st_uid).pw_name @@ -163,7 +162,7 @@ class ActionModule(ActionBase): if self._play_context.diff: diff = self._get_diff_data(dest, resultant, task_vars, source_file=False) - if not self._play_context.check_mode: # do actual work thorugh copy + if not self._play_context.check_mode: # do actual work through copy xfered = self._transfer_data(self._connection._shell.join_path(tmp, 'source'), resultant) # fix file permissions when the copy is done as a different user @@ -176,7 +175,7 @@ class ActionModule(ActionBase): dest=dest, original_basename=os.path.basename(source), follow=True, - ), + ), ) result.update(self._execute_module(module_name='copy', module_args=new_module_args, task_vars=task_vars, tmp=tmp, delete_remote_tmp=False)) diff --git a/lib/ansible/plugins/action/unarchive.py b/lib/ansible/plugins/action/unarchive.py index 5884cb4a95..70cde2e97d 100644 --- a/lib/ansible/plugins/action/unarchive.py +++ b/lib/ansible/plugins/action/unarchive.py @@ -20,10 +20,11 @@ __metaclass__ = type import os +from ansible.errors import AnsibleError +from ansible.module_utils._text import to_native from ansible.plugins.action import ActionBase from ansible.utils.boolean import boolean -from ansible.errors import AnsibleError -from ansible.utils.unicode import to_str + class ActionModule(ActionBase): @@ -82,7 +83,7 @@ class ActionModule(ActionBase): source = self._loader.get_real_file(self._find_needle('files', source)) except AnsibleError as e: result['failed'] = True - result['msg'] = to_str(e) + result['msg'] = to_native(e) self._remove_tmp_path(tmp) return result diff --git a/lib/ansible/plugins/action/win_reboot.py b/lib/ansible/plugins/action/win_reboot.py index ea2021be70..48056c32db 100644 --- a/lib/ansible/plugins/action/win_reboot.py +++ b/lib/ansible/plugins/action/win_reboot.py @@ -19,26 +19,24 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type -from ansible.plugins.action import ActionBase -from ansible.utils.boolean import boolean -from ansible.utils.unicode import to_unicode -from ansible.errors import AnsibleUndefinedVariable - import socket import time -import traceback from datetime import datetime, timedelta +from ansible.plugins.action import ActionBase + try: from __main__ import display except ImportError: from ansible.utils.display import Display display = Display() + class TimedOutException(Exception): pass + class ActionModule(ActionBase): TRANSFERS_FILES = False @@ -50,7 +48,7 @@ class ActionModule(ActionBase): def do_until_success_or_timeout(self, what, timeout_sec, what_desc, fail_sleep_sec=1): max_end_time = datetime.utcnow() + timedelta(seconds=timeout_sec) - + while datetime.utcnow() < max_end_time: try: what() @@ -82,7 +80,7 @@ class ActionModule(ActionBase): winrm_port = self._connection._winrm_port result = super(ActionModule, self).run(tmp, task_vars) - + # initiate reboot (rc, stdout, stderr) = self._connection.exec_command("shutdown /r /t %d" % pre_reboot_delay_sec) @@ -92,7 +90,7 @@ class ActionModule(ActionBase): result['msg'] = "Shutdown command failed, error text was %s" % stderr return result - def raise_if_port_open(): + def raise_if_port_open(): try: sock = socket.create_connection((winrm_host, winrm_port), connect_timeout_sec) sock.close() @@ -137,4 +135,3 @@ class ActionModule(ActionBase): result['msg'] = toex.message return result - diff --git a/lib/ansible/plugins/cache/jsonfile.py b/lib/ansible/plugins/cache/jsonfile.py index 1ee183e094..f2f42f3cc2 100644 --- a/lib/ansible/plugins/cache/jsonfile.py +++ b/lib/ansible/plugins/cache/jsonfile.py @@ -31,9 +31,9 @@ except ImportError: from ansible import constants as C from ansible.errors import AnsibleError +from ansible.module_utils._text import to_bytes from ansible.parsing.utils.jsonify import jsonify from ansible.plugins.cache.base import BaseCacheModule -from ansible.utils.unicode import to_bytes try: from __main__ import display diff --git a/lib/ansible/plugins/callback/__init__.py b/lib/ansible/plugins/callback/__init__.py index 19ced6d49a..d02e5ac3be 100644 --- a/lib/ansible/plugins/callback/__init__.py +++ b/lib/ansible/plugins/callback/__init__.py @@ -24,12 +24,10 @@ import difflib import warnings from copy import deepcopy -from ansible.compat.six import string_types - from ansible import constants as C -from ansible.vars import strip_internal_keys +from ansible.module_utils._text import to_text from ansible.utils.color import stringc -from ansible.utils.unicode import to_unicode +from ansible.vars import strip_internal_keys try: from __main__ import display as global_display @@ -37,14 +35,15 @@ except ImportError: from ansible.utils.display import Display global_display = Display() -__all__ = ["CallbackBase"] - try: from __main__ import cli except ImportError: - # using API w/o cli + # using API w/o cli cli = False +__all__ = ["CallbackBase"] + + class CallbackBase: ''' @@ -146,8 +145,8 @@ class CallbackBase: after_header = "after: %s" % diff['after_header'] else: after_header = 'after' - differ = difflib.unified_diff(to_unicode(diff['before']).splitlines(True), - to_unicode(diff['after']).splitlines(True), + differ = difflib.unified_diff(to_text(diff['before']).splitlines(True), + to_text(diff['after']).splitlines(True), fromfile=before_header, tofile=after_header, fromfiledate='', @@ -166,7 +165,7 @@ class CallbackBase: if has_diff: ret.append('\n') if 'prepared' in diff: - ret.append(to_unicode(diff['prepared'])) + ret.append(to_text(diff['prepared'])) except UnicodeDecodeError: ret.append(">> the files are different, but the diff library cannot compare unicode strings\n\n") return u''.join(ret) @@ -362,4 +361,3 @@ class CallbackBase: def v2_runner_retry(self, result): pass - diff --git a/lib/ansible/plugins/callback/junit.py b/lib/ansible/plugins/callback/junit.py index 067c7551b0..9ce0dfa172 100644 --- a/lib/ansible/plugins/callback/junit.py +++ b/lib/ansible/plugins/callback/junit.py @@ -21,8 +21,8 @@ __metaclass__ = type import os import time +from ansible.module_utils._text import to_bytes from ansible.plugins.callback import CallbackBase -from ansible.utils.unicode import to_bytes try: from junit_xml import TestSuite, TestCase @@ -40,6 +40,7 @@ except ImportError: except ImportError: HAS_ORDERED_DICT = False + class CallbackModule(CallbackBase): """ This callback writes playbook output to a JUnit formatted XML file. @@ -181,7 +182,7 @@ class CallbackModule(CallbackBase): output_file = os.path.join(self._output_dir, '%s-%s.xml' % (self._playbook_name, time.time())) with open(output_file, 'wb') as xml: - xml.write(to_bytes(report, errors='strict')) + xml.write(to_bytes(report, errors='surrogate_or_strict')) def v2_playbook_on_start(self, playbook): self._playbook_path = playbook._file_name diff --git a/lib/ansible/plugins/callback/log_plays.py b/lib/ansible/plugins/callback/log_plays.py index 7b708a74ba..f82ebda9e1 100644 --- a/lib/ansible/plugins/callback/log_plays.py +++ b/lib/ansible/plugins/callback/log_plays.py @@ -23,9 +23,10 @@ import os import time import json -from ansible.utils.unicode import to_bytes +from ansible.module_utils._text import to_bytes from ansible.plugins.callback import CallbackBase + # NOTE: in Ansible 1.2 or later general logging is available without # this plugin, just set ANSIBLE_LOG_PATH as an environment variable # or log_path in the DEFAULTS section of your ansible configuration diff --git a/lib/ansible/plugins/callback/mail.py b/lib/ansible/plugins/callback/mail.py index 3169b4c6e3..65bdc5bbbe 100644 --- a/lib/ansible/plugins/callback/mail.py +++ b/lib/ansible/plugins/callback/mail.py @@ -25,9 +25,10 @@ import smtplib import json from ansible.compat.six import string_types -from ansible.utils.unicode import to_bytes +from ansible.module_utils._text import to_bytes from ansible.plugins.callback import CallbackBase + def mail(subject='Ansible error mail', sender=None, to=None, cc=None, bcc=None, body=None, smtphost=None): if sender is None: @@ -84,7 +85,7 @@ class CallbackModule(CallbackBase): if ignore_errors: return sender = '"Ansible: %s" <root>' % host - attach = res._task.action + attach = res._task.action if 'invocation' in res._result: attach = "%s: %s" % (res._result['invocation']['module_name'], json.dumps(res._result['invocation']['module_args'])) diff --git a/lib/ansible/plugins/callback/tree.py b/lib/ansible/plugins/callback/tree.py index f067197cbc..e403d99fd2 100644 --- a/lib/ansible/plugins/callback/tree.py +++ b/lib/ansible/plugins/callback/tree.py @@ -20,10 +20,10 @@ __metaclass__ = type import os +from ansible.constants import TREE_DIR +from ansible.module_utils._text import to_bytes from ansible.plugins.callback import CallbackBase from ansible.utils.path import makedirs_safe -from ansible.utils.unicode import to_bytes -from ansible.constants import TREE_DIR class CallbackModule(CallbackBase): @@ -68,4 +68,3 @@ class CallbackModule(CallbackBase): def v2_runner_on_unreachable(self, result): self.result_to_tree(result) - diff --git a/lib/ansible/plugins/connection/__init__.py b/lib/ansible/plugins/connection/__init__.py index edd6b498e8..43921822e4 100644 --- a/lib/ansible/plugins/connection/__init__.py +++ b/lib/ansible/plugins/connection/__init__.py @@ -1,4 +1,3 @@ - # (c) 2015 Toshio Kuratomi <tkuratomi@ansible.com> # # This file is part of Ansible @@ -32,8 +31,9 @@ from ansible.compat.six import with_metaclass from ansible import constants as C from ansible.compat.six import string_types from ansible.errors import AnsibleError +from ansible.module_utils._text import to_bytes, to_text from ansible.plugins import shell_loader -from ansible.utils.unicode import to_bytes, to_unicode + try: from __main__ import display @@ -138,9 +138,9 @@ class ConnectionBase(with_metaclass(ABCMeta, object)): # exception, it merely mangles the output: # >>> shlex.split(u't e') # ['t\x00\x00\x00', '\x00\x00\x00e\x00\x00\x00'] - return [to_unicode(x.strip()) for x in shlex.split(to_bytes(argstring)) if x.strip()] + return [to_text(x.strip()) for x in shlex.split(to_bytes(argstring)) if x.strip()] except AttributeError: - return [to_unicode(x.strip()) for x in shlex.split(argstring) if x.strip()] + return [to_text(x.strip()) for x in shlex.split(argstring) if x.strip()] @abstractproperty def transport(self): diff --git a/lib/ansible/plugins/connection/accelerate.py b/lib/ansible/plugins/connection/accelerate.py index 110e2bb2da..757350467a 100644 --- a/lib/ansible/plugins/connection/accelerate.py +++ b/lib/ansible/plugins/connection/accelerate.py @@ -27,10 +27,11 @@ import time from ansible import constants as C from ansible.errors import AnsibleError, AnsibleFileNotFound, AnsibleConnectionFailure +from ansible.module_utils._text import to_bytes from ansible.parsing.utils.jsonify import jsonify from ansible.plugins.connection import ConnectionBase from ansible.utils.encrypt import key_for_hostname, keyczar_encrypt, keyczar_decrypt -from ansible.utils.unicode import to_bytes + try: from __main__ import display @@ -211,7 +212,7 @@ class Connection(ConnectionBase): ''' transfer a file from local to remote ''' display.vvv("PUT %s TO %s" % (in_path, out_path), host=self._play_context.remote_addr) - in_path = to_bytes(in_path, errors='strict') + in_path = to_bytes(in_path, errors='surrogate_or_strict') if not os.path.exists(in_path): raise AnsibleFileNotFound("file or module does not exist: %s" % in_path) @@ -265,7 +266,7 @@ class Connection(ConnectionBase): if self.send_data(data): raise AnsibleError("failed to initiate the file fetch with %s" % self._play_context.remote_addr) - fh = open(to_bytes(out_path, errors='strict'), "w") + fh = open(to_bytes(out_path, errors='surrogate_or_strict'), "w") try: bytes = 0 while True: diff --git a/lib/ansible/plugins/connection/chroot.py b/lib/ansible/plugins/connection/chroot.py index d9f499a11c..d067027a91 100644 --- a/lib/ansible/plugins/connection/chroot.py +++ b/lib/ansible/plugins/connection/chroot.py @@ -28,9 +28,10 @@ import traceback from ansible import constants as C from ansible.errors import AnsibleError -from ansible.plugins.connection import ConnectionBase, BUFSIZE from ansible.module_utils.basic import is_executable -from ansible.utils.unicode import to_bytes +from ansible.module_utils._text import to_bytes +from ansible.plugins.connection import ConnectionBase, BUFSIZE + try: from __main__ import display @@ -93,7 +94,7 @@ class Connection(ConnectionBase): local_cmd = [self.chroot_cmd, self.chroot, executable, '-c', cmd] display.vvv("EXEC %s" % (local_cmd), host=self.chroot) - local_cmd = [to_bytes(i, errors='strict') for i in local_cmd] + local_cmd = [to_bytes(i, errors='surrogate_or_strict') for i in local_cmd] p = subprocess.Popen(local_cmd, shell=False, stdin=stdin, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -129,7 +130,7 @@ class Connection(ConnectionBase): out_path = pipes.quote(self._prefix_login_path(out_path)) try: - with open(to_bytes(in_path, errors='strict'), 'rb') as in_file: + with open(to_bytes(in_path, errors='surrogate_or_strict'), 'rb') as in_file: try: p = self._buffered_exec_command('dd of=%s bs=%s' % (out_path, BUFSIZE), stdin=in_file) except OSError: @@ -155,7 +156,7 @@ class Connection(ConnectionBase): except OSError: raise AnsibleError("chroot connection requires dd command in the chroot") - with open(to_bytes(out_path, errors='strict'), 'wb+') as out_file: + with open(to_bytes(out_path, errors='surrogate_or_strict'), 'wb+') as out_file: try: chunk = p.stdout.read(BUFSIZE) while chunk: diff --git a/lib/ansible/plugins/connection/docker.py b/lib/ansible/plugins/connection/docker.py index 7db04cba3e..f4b991ad51 100644 --- a/lib/ansible/plugins/connection/docker.py +++ b/lib/ansible/plugins/connection/docker.py @@ -35,8 +35,9 @@ from distutils.version import LooseVersion import ansible.constants as C from ansible.errors import AnsibleError, AnsibleFileNotFound +from ansible.module_utils._text import to_bytes from ansible.plugins.connection import ConnectionBase, BUFSIZE -from ansible.utils.unicode import to_bytes + try: from __main__ import display @@ -196,7 +197,7 @@ class Connection(ConnectionBase): local_cmd = self._build_exec_cmd([self._play_context.executable, '-c', cmd]) display.vvv("EXEC %s" % (local_cmd,), host=self._play_context.remote_addr) - local_cmd = [to_bytes(i, errors='strict') for i in local_cmd] + local_cmd = [to_bytes(i, errors='surrogate_or_strict') for i in local_cmd] p = subprocess.Popen(local_cmd, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -223,7 +224,7 @@ class Connection(ConnectionBase): display.vvv("PUT %s TO %s" % (in_path, out_path), host=self._play_context.remote_addr) out_path = self._prefix_login_path(out_path) - if not os.path.exists(to_bytes(in_path, errors='strict')): + if not os.path.exists(to_bytes(in_path, errors='surrogate_or_strict')): raise AnsibleFileNotFound( "file or module does not exist: %s" % in_path) @@ -233,8 +234,8 @@ class Connection(ConnectionBase): # Although docker version 1.8 and later provide support, the # owner and group of the files are always set to root args = self._build_exec_cmd([self._play_context.executable, "-c", "dd of=%s bs=%s" % (out_path, BUFSIZE)]) - args = [to_bytes(i, errors='strict') for i in args] - with open(to_bytes(in_path, errors='strict'), 'rb') as in_file: + args = [to_bytes(i, errors='surrogate_or_strict') for i in args] + with open(to_bytes(in_path, errors='surrogate_or_strict'), 'rb') as in_file: try: p = subprocess.Popen(args, stdin=in_file, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -256,7 +257,7 @@ class Connection(ConnectionBase): out_dir = os.path.dirname(out_path) args = [self.docker_cmd, "cp", "%s:%s" % (self._play_context.remote_addr, in_path), out_dir] - args = [to_bytes(i, errors='strict') for i in args] + args = [to_bytes(i, errors='surrogate_or_strict') for i in args] p = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) diff --git a/lib/ansible/plugins/connection/jail.py b/lib/ansible/plugins/connection/jail.py index ac2cdf9891..93936a0a78 100644 --- a/lib/ansible/plugins/connection/jail.py +++ b/lib/ansible/plugins/connection/jail.py @@ -27,10 +27,9 @@ import pipes import subprocess import traceback -from ansible import constants as C from ansible.errors import AnsibleError +from ansible.module_utils._text import to_bytes from ansible.plugins.connection import ConnectionBase, BUFSIZE -from ansible.utils.unicode import to_bytes try: from __main__ import display @@ -117,7 +116,7 @@ class Connection(ConnectionBase): local_cmd += [self.jail, self._play_context.executable, '-c', set_env + cmd] display.vvv("EXEC %s" % (local_cmd,), host=self.jail) - local_cmd = [to_bytes(i, errors='strict') for i in local_cmd] + local_cmd = [to_bytes(i, errors='surrogate_or_strict') for i in local_cmd] p = subprocess.Popen(local_cmd, shell=False, stdin=stdin, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -153,7 +152,7 @@ class Connection(ConnectionBase): out_path = pipes.quote(self._prefix_login_path(out_path)) try: - with open(to_bytes(in_path, errors='strict'), 'rb') as in_file: + with open(to_bytes(in_path, errors='surrogate_or_strict'), 'rb') as in_file: try: p = self._buffered_exec_command('dd of=%s bs=%s' % (out_path, BUFSIZE), stdin=in_file) except OSError: @@ -179,7 +178,7 @@ class Connection(ConnectionBase): except OSError: raise AnsibleError("jail connection requires dd command in the jail") - with open(to_bytes(out_path, errors='strict'), 'wb+') as out_file: + with open(to_bytes(out_path, errors='surrogate_or_strict'), 'wb+') as out_file: try: chunk = p.stdout.read(BUFSIZE) while chunk: diff --git a/lib/ansible/plugins/connection/libvirt_lxc.py b/lib/ansible/plugins/connection/libvirt_lxc.py index 2b92e9c9dc..4ccb66f556 100644 --- a/lib/ansible/plugins/connection/libvirt_lxc.py +++ b/lib/ansible/plugins/connection/libvirt_lxc.py @@ -29,8 +29,9 @@ import traceback from ansible import constants as C from ansible.errors import AnsibleError +from ansible.module_utils._text import to_bytes from ansible.plugins.connection import ConnectionBase, BUFSIZE -from ansible.utils.unicode import to_bytes + try: from __main__ import display @@ -94,7 +95,7 @@ class Connection(ConnectionBase): local_cmd += [self.lxc, '--', executable, '-c', cmd] display.vvv("EXEC %s" % (local_cmd,), host=self.lxc) - local_cmd = [to_bytes(i, errors='strict') for i in local_cmd] + local_cmd = [to_bytes(i, errors='surrogate_or_strict') for i in local_cmd] p = subprocess.Popen(local_cmd, shell=False, stdin=stdin, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -130,7 +131,7 @@ class Connection(ConnectionBase): out_path = pipes.quote(self._prefix_login_path(out_path)) try: - with open(to_bytes(in_path, errors='strict'), 'rb') as in_file: + with open(to_bytes(in_path, errors='surrogate_or_strict'), 'rb') as in_file: try: p = self._buffered_exec_command('dd of=%s bs=%s' % (out_path, BUFSIZE), stdin=in_file) except OSError: @@ -156,7 +157,7 @@ class Connection(ConnectionBase): except OSError: raise AnsibleError("chroot connection requires dd command in the chroot") - with open(to_bytes(out_path, errors='strict'), 'wb+') as out_file: + with open(to_bytes(out_path, errors='surrogate_or_strict'), 'wb+') as out_file: try: chunk = p.stdout.read(BUFSIZE) while chunk: diff --git a/lib/ansible/plugins/connection/local.py b/lib/ansible/plugins/connection/local.py index 0c7bb5ccb7..728522acda 100644 --- a/lib/ansible/plugins/connection/local.py +++ b/lib/ansible/plugins/connection/local.py @@ -30,8 +30,9 @@ from ansible.compat.six import text_type, binary_type import ansible.constants as C from ansible.errors import AnsibleError, AnsibleFileNotFound +from ansible.module_utils._text import to_bytes, to_native from ansible.plugins.connection import ConnectionBase -from ansible.utils.unicode import to_bytes, to_str + try: from __main__ import display @@ -122,14 +123,14 @@ class Connection(ConnectionBase): super(Connection, self).put_file(in_path, out_path) display.vvv(u"PUT {0} TO {1}".format(in_path, out_path), host=self._play_context.remote_addr) - if not os.path.exists(to_bytes(in_path, errors='strict')): - raise AnsibleFileNotFound("file or module does not exist: {0}".format(to_str(in_path))) + if not os.path.exists(to_bytes(in_path, errors='surrogate_or_strict')): + raise AnsibleFileNotFound("file or module does not exist: {0}".format(to_native(in_path))) try: - shutil.copyfile(to_bytes(in_path, errors='strict'), to_bytes(out_path, errors='strict')) + shutil.copyfile(to_bytes(in_path, errors='surrogate_or_strict'), to_bytes(out_path, errors='surrogate_or_strict')) except shutil.Error: - raise AnsibleError("failed to copy: {0} and {1} are the same".format(to_str(in_path), to_str(out_path))) + raise AnsibleError("failed to copy: {0} and {1} are the same".format(to_native(in_path), to_native(out_path))) except IOError as e: - raise AnsibleError("failed to transfer file to {0}: {1}".format(to_str(out_path), to_str(e))) + raise AnsibleError("failed to transfer file to {0}: {1}".format(to_native(out_path), to_native(e))) def fetch_file(self, in_path, out_path): ''' fetch a file from local to local -- for copatibility ''' diff --git a/lib/ansible/plugins/connection/lxc.py b/lib/ansible/plugins/connection/lxc.py index 36c87bb593..34cb66c484 100644 --- a/lib/ansible/plugins/connection/lxc.py +++ b/lib/ansible/plugins/connection/lxc.py @@ -24,10 +24,6 @@ import traceback import select import fcntl import errno -from ansible import errors -from ansible import constants as C -from ansible.plugins.connection import ConnectionBase -from ansible.utils.unicode import to_bytes HAS_LIBLXC = False try: @@ -36,6 +32,12 @@ try: except ImportError: pass +from ansible import constants as C +from ansible import errors +from ansible.module_utils._text import to_bytes +from ansible.plugins.connection import ConnectionBase + + class Connection(ConnectionBase): ''' Local lxc based connections ''' @@ -102,8 +104,8 @@ class Connection(ConnectionBase): ''' run a command on the chroot ''' super(Connection, self).exec_command(cmd, in_data=in_data, sudoable=sudoable) - executable = to_bytes(self._play_context.executable, errors='strict') - local_cmd = [executable, '-c', to_bytes(cmd, errors='strict')] + executable = to_bytes(self._play_context.executable, errors='surrogate_or_strict') + local_cmd = [executable, '-c', to_bytes(cmd, errors='surrogate_or_strict')] read_stdout, write_stdout = None, None read_stderr, write_stderr = None, None @@ -154,8 +156,8 @@ class Connection(ConnectionBase): ''' transfer a file from local to lxc ''' super(Connection, self).put_file(in_path, out_path) self._display.vvv("PUT %s TO %s" % (in_path, out_path), host=self.container_name) - in_path = to_bytes(in_path, errors='strict') - out_path = to_bytes(out_path, errors='strict') + in_path = to_bytes(in_path, errors='surrogate_or_strict') + out_path = to_bytes(out_path, errors='surrogate_or_strict') if not os.path.exists(in_path): msg = "file or module does not exist: %s" % in_path @@ -182,8 +184,8 @@ class Connection(ConnectionBase): ''' fetch a file from lxc to local ''' super(Connection, self).fetch_file(in_path, out_path) self._display.vvv("FETCH %s TO %s" % (in_path, out_path), host=self.container_name) - in_path = to_bytes(in_path, errors='strict') - out_path = to_bytes(out_path, errors='strict') + in_path = to_bytes(in_path, errors='surrogate_or_strict') + out_path = to_bytes(out_path, errors='surrogate_or_strict') try: dst_file = open(out_path, "wb") diff --git a/lib/ansible/plugins/connection/lxd.py b/lib/ansible/plugins/connection/lxd.py index 8e225c973c..d822bcfa21 100644 --- a/lib/ansible/plugins/connection/lxd.py +++ b/lib/ansible/plugins/connection/lxd.py @@ -23,8 +23,8 @@ from distutils.spawn import find_executable from subprocess import call, Popen, PIPE from ansible.errors import AnsibleError, AnsibleConnectionFailure, AnsibleFileNotFound +from ansible.module_utils._text import to_bytes, to_text from ansible.plugins.connection import ConnectionBase -from ansible.utils.unicode import to_bytes, to_unicode class Connection(ConnectionBase): @@ -61,14 +61,14 @@ class Connection(ConnectionBase): local_cmd = [self._lxc_cmd, "exec", self._host, "--", self._play_context.executable, "-c", cmd] - local_cmd = [to_bytes(i, errors='strict') for i in local_cmd] - in_data = to_bytes(in_data, errors='strict', nonstring='passthru') + local_cmd = [to_bytes(i, errors='surrogate_or_strict') for i in local_cmd] + in_data = to_bytes(in_data, errors='surrogate_or_strict', nonstring='passthru') process = Popen(local_cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) stdout, stderr = process.communicate(in_data) - stdout = to_unicode(stdout) - stderr = to_unicode(stderr) + stdout = to_text(stdout) + stderr = to_text(stderr) if stderr == "error: Container is not running.\n": raise AnsibleConnectionFailure("container not running: %s" % self._host) @@ -84,12 +84,12 @@ class Connection(ConnectionBase): self._display.vvv(u"PUT {0} TO {1}".format(in_path, out_path), host=self._host) - if not os.path.isfile(to_bytes(in_path, errors='strict')): + if not os.path.isfile(to_bytes(in_path, errors='surrogate_or_strict')): raise AnsibleFileNotFound("input path is not a file: %s" % in_path) local_cmd = [self._lxc_cmd, "file", "push", in_path, self._host + "/" + out_path] - local_cmd = [to_bytes(i, errors='strict') for i in local_cmd] + local_cmd = [to_bytes(i, errors='surrogate_or_strict') for i in local_cmd] call(local_cmd) @@ -101,7 +101,7 @@ class Connection(ConnectionBase): local_cmd = [self._lxc_cmd, "file", "pull", self._host + "/" + in_path, out_path] - local_cmd = [to_bytes(i, errors='strict') for i in local_cmd] + local_cmd = [to_bytes(i, errors='surrogate_or_strict') for i in local_cmd] call(local_cmd) diff --git a/lib/ansible/plugins/connection/paramiko_ssh.py b/lib/ansible/plugins/connection/paramiko_ssh.py index 2f8226e35d..4fdf0fe4bd 100644 --- a/lib/ansible/plugins/connection/paramiko_ssh.py +++ b/lib/ansible/plugins/connection/paramiko_ssh.py @@ -44,7 +44,7 @@ from ansible.compat.six.moves import input from ansible.errors import AnsibleError, AnsibleConnectionFailure, AnsibleFileNotFound from ansible.plugins.connection import ConnectionBase from ansible.utils.path import makedirs_safe -from ansible.utils.unicode import to_bytes +from ansible.module_utils._text import to_bytes try: from __main__ import display @@ -52,6 +52,7 @@ except ImportError: from ansible.utils.display import Display display = Display() + AUTHENTICITY_MSG=""" paramiko: The authenticity of host '%s' can't be established. The %s key fingerprint is %s. @@ -273,7 +274,7 @@ class Connection(ConnectionBase): display.vvv("EXEC %s" % cmd, host=self._play_context.remote_addr) - cmd = to_bytes(cmd, errors='strict') + cmd = to_bytes(cmd, errors='surrogate_or_strict') no_prompt_out = b'' no_prompt_err = b'' @@ -330,7 +331,7 @@ class Connection(ConnectionBase): display.vvv("PUT %s TO %s" % (in_path, out_path), host=self._play_context.remote_addr) - if not os.path.exists(to_bytes(in_path, errors='strict')): + if not os.path.exists(to_bytes(in_path, errors='surrogate_or_strict')): raise AnsibleFileNotFound("file or module does not exist: %s" % in_path) try: @@ -339,7 +340,7 @@ class Connection(ConnectionBase): raise AnsibleError("failed to open a SFTP connection (%s)" % e) try: - self.sftp.put(to_bytes(in_path, errors='strict'), to_bytes(out_path, errors='strict')) + self.sftp.put(to_bytes(in_path, errors='surrogate_or_strict'), to_bytes(out_path, errors='surrogate_or_strict')) except IOError: raise AnsibleError("failed to transfer file to %s" % out_path) @@ -365,7 +366,7 @@ class Connection(ConnectionBase): raise AnsibleError("failed to open a SFTP connection (%s)", e) try: - self.sftp.get(to_bytes(in_path, errors='strict'), to_bytes(out_path, errors='strict')) + self.sftp.get(to_bytes(in_path, errors='surrogate_or_strict'), to_bytes(out_path, errors='surrogate_or_strict')) except IOError: raise AnsibleError("failed to transfer file from %s" % in_path) diff --git a/lib/ansible/plugins/connection/ssh.py b/lib/ansible/plugins/connection/ssh.py index def3f85403..fd99ccdef0 100644 --- a/lib/ansible/plugins/connection/ssh.py +++ b/lib/ansible/plugins/connection/ssh.py @@ -29,11 +29,11 @@ import subprocess import time from ansible import constants as C +from ansible.compat.six import text_type, binary_type from ansible.errors import AnsibleError, AnsibleConnectionFailure, AnsibleFileNotFound +from ansible.module_utils._text import to_bytes, to_native, to_text from ansible.plugins.connection import ConnectionBase from ansible.utils.path import unfrackpath, makedirs_safe -from ansible.utils.unicode import to_bytes, to_unicode, to_str -from ansible.compat.six import text_type, binary_type try: from __main__ import display @@ -107,7 +107,7 @@ class Connection(ConnectionBase): explanation of why they were added. """ self._command += args - display.vvvvv('SSH: ' + explanation + ': (%s)' % ')('.join(map(to_unicode, args)), host=self._play_context.remote_addr) + display.vvvvv('SSH: ' + explanation + ': (%s)' % ')('.join(map(to_text, args)), host=self._play_context.remote_addr) def _build_command(self, binary, *other_args): ''' @@ -222,7 +222,7 @@ class Connection(ConnectionBase): # The directory must exist and be writable. makedirs_safe(b_cpdir, 0o700) if not os.access(b_cpdir, os.W_OK): - raise AnsibleError("Cannot write to ControlPath %s" % to_str(cpdir)) + raise AnsibleError("Cannot write to ControlPath %s" % to_native(cpdir)) args = ("-o", "ControlPath=" + C.ANSIBLE_SSH_CONTROL_PATH % dict(directory=cpdir)) self._add_args("found only ControlPersist; added ControlPath", args) @@ -275,7 +275,7 @@ class Connection(ConnectionBase): output = [] for b_line in b_chunk.splitlines(True): - display_line = to_unicode(b_line, errors='replace').rstrip('\r\n') + display_line = to_text(b_line).rstrip('\r\n') suppress_output = False #display.debug("Examining line (source=%s, state=%s): '%s'" % (source, state, display_line)) @@ -314,7 +314,7 @@ class Connection(ConnectionBase): Starts the command and communicates with it until it ends. ''' - display_cmd = list(map(pipes.quote, map(to_unicode, cmd))) + display_cmd = list(map(pipes.quote, map(to_text, cmd))) display.vvv(u'SSH: EXEC {0}'.format(u' '.join(display_cmd)), host=self.host) # Start the given command. If we don't need to pipeline data, we can try @@ -424,7 +424,7 @@ class Connection(ConnectionBase): if p.poll() is not None: break self._terminate_process(p) - raise AnsibleError('Timeout (%ds) waiting for privilege escalation prompt: %s' % (timeout, to_str(b_stdout))) + raise AnsibleError('Timeout (%ds) waiting for privilege escalation prompt: %s' % (timeout, to_native(b_stdout))) # Read whatever output is available on stdout and stderr, and stop # listening to the pipe if it's been closed. @@ -434,14 +434,14 @@ class Connection(ConnectionBase): if b_chunk == b'': rpipes.remove(p.stdout) b_tmp_stdout += b_chunk - display.debug("stdout chunk (state=%s):\n>>>%s<<<\n" % (state, to_unicode(b_chunk, errors='replace'))) + display.debug("stdout chunk (state=%s):\n>>>%s<<<\n" % (state, to_text(b_chunk))) if p.stderr in rfd: b_chunk = p.stderr.read() if b_chunk == b'': rpipes.remove(p.stderr) b_tmp_stderr += b_chunk - display.debug("stderr chunk (state=%s):\n>>>%s<<<\n" % (state, to_unicode(b_chunk, errors='replace'))) + display.debug("stderr chunk (state=%s):\n>>>%s<<<\n" % (state, to_text(b_chunk))) # We examine the output line-by-line until we have negotiated any # privilege escalation prompt and subsequent success/error message. @@ -631,8 +631,8 @@ class Connection(ConnectionBase): super(Connection, self).put_file(in_path, out_path) display.vvv(u"PUT {0} TO {1}".format(in_path, out_path), host=self.host) - if not os.path.exists(to_bytes(in_path, errors='strict')): - raise AnsibleFileNotFound("file or module does not exist: {0}".format(to_str(in_path))) + if not os.path.exists(to_bytes(in_path, errors='surrogate_or_strict')): + raise AnsibleFileNotFound("file or module does not exist: {0}".format(to_native(in_path))) # scp and sftp require square brackets for IPv6 addresses, but # accept them for hostnames and IPv4 addresses too. @@ -649,7 +649,7 @@ class Connection(ConnectionBase): (returncode, stdout, stderr) = self._run(cmd, in_data) if returncode != 0: - raise AnsibleError("failed to transfer file to {0}:\n{1}\n{2}".format(to_str(out_path), to_str(stdout), to_str(stderr))) + raise AnsibleError("failed to transfer file to {0}:\n{1}\n{2}".format(to_native(out_path), to_native(stdout), to_native(stderr))) def fetch_file(self, in_path, out_path): ''' fetch a file from remote to local ''' diff --git a/lib/ansible/plugins/connection/winrm.py b/lib/ansible/plugins/connection/winrm.py index 670d878cdd..fbc31ae4ed 100644 --- a/lib/ansible/plugins/connection/winrm.py +++ b/lib/ansible/plugins/connection/winrm.py @@ -26,9 +26,21 @@ import shlex import traceback import json +HAVE_KERBEROS = False +try: + import kerberos + HAVE_KERBEROS = True +except ImportError: + pass + from ansible.compat.six import string_types from ansible.compat.six.moves.urllib.parse import urlunsplit from ansible.errors import AnsibleError, AnsibleConnectionFailure +from ansible.errors import AnsibleFileNotFound +from ansible.module_utils._text import to_bytes, to_native, to_text +from ansible.plugins.connection import ConnectionBase +from ansible.utils.hashing import secure_hash +from ansible.utils.path import makedirs_safe try: import winrm @@ -42,25 +54,13 @@ try: except ImportError: raise AnsibleError("xmltodict is not installed") -HAVE_KERBEROS = False -try: - import kerberos - HAVE_KERBEROS = True -except ImportError: - pass - -from ansible.errors import AnsibleFileNotFound -from ansible.plugins.connection import ConnectionBase -from ansible.utils.hashing import secure_hash -from ansible.utils.path import makedirs_safe -from ansible.utils.unicode import to_bytes, to_unicode, to_str - try: from __main__ import display except ImportError: from ansible.utils.display import Display display = Display() + class Connection(ConnectionBase): '''WinRM connections over HTTP/HTTPS.''' @@ -156,10 +156,10 @@ class Connection(ConnectionBase): return protocol except Exception as e: - err_msg = to_unicode(e).strip() - if re.search(to_unicode(r'Operation\s+?timed\s+?out'), err_msg, re.I): + err_msg = to_text(e).strip() + if re.search(to_text(r'Operation\s+?timed\s+?out'), err_msg, re.I): raise AnsibleError('the connection attempt timed out') - m = re.search(to_unicode(r'Code\s+?(\d{3})'), err_msg) + m = re.search(to_text(r'Code\s+?(\d{3})'), err_msg) if m: code = int(m.groups()[0]) if code == 401: @@ -167,9 +167,9 @@ class Connection(ConnectionBase): elif code == 411: return protocol errors.append(u'%s: %s' % (transport, err_msg)) - display.vvvvv(u'WINRM CONNECTION ERROR: %s\n%s' % (err_msg, to_unicode(traceback.format_exc())), host=self._winrm_host) + display.vvvvv(u'WINRM CONNECTION ERROR: %s\n%s' % (err_msg, to_text(traceback.format_exc())), host=self._winrm_host) if errors: - raise AnsibleConnectionFailure(', '.join(map(to_str, errors))) + raise AnsibleConnectionFailure(', '.join(map(to_native, errors))) else: raise AnsibleError('No transport found for WinRM connection') @@ -220,12 +220,12 @@ class Connection(ConnectionBase): # TODO: check result from response and set stdin_push_failed if we have nonzero if from_exec: - display.vvvvv('WINRM RESULT %r' % to_unicode(response), host=self._winrm_host) + display.vvvvv('WINRM RESULT %r' % to_text(response), host=self._winrm_host) else: - display.vvvvvv('WINRM RESULT %r' % to_unicode(response), host=self._winrm_host) + display.vvvvvv('WINRM RESULT %r' % to_text(response), host=self._winrm_host) - display.vvvvvv('WINRM STDOUT %s' % to_unicode(response.std_out), host=self._winrm_host) - display.vvvvvv('WINRM STDERR %s' % to_unicode(response.std_err), host=self._winrm_host) + display.vvvvvv('WINRM STDOUT %s' % to_text(response.std_out), host=self._winrm_host) + display.vvvvvv('WINRM STDERR %s' % to_text(response.std_err), host=self._winrm_host) if stdin_push_failed: raise AnsibleError('winrm send_input failed; \nstdout: %s\nstderr %s' % (response.std_out, response.std_err)) @@ -250,7 +250,7 @@ class Connection(ConnectionBase): def exec_command(self, cmd, in_data=None, sudoable=True): super(Connection, self).exec_command(cmd, in_data=in_data, sudoable=sudoable) cmd_parts = shlex.split(to_bytes(cmd), posix=False) - cmd_parts = map(to_unicode, cmd_parts) + cmd_parts = map(to_text, cmd_parts) script = None cmd_ext = cmd_parts and self._shell._unquote(cmd_parts[0]).lower()[-4:] or '' # Support running .ps1 files (via script/raw). @@ -266,7 +266,7 @@ class Connection(ConnectionBase): cmd_parts = self._shell._encode_script(script, as_list=True, strict_mode=False) if '-EncodedCommand' in cmd_parts: encoded_cmd = cmd_parts[cmd_parts.index('-EncodedCommand') + 1] - decoded_cmd = to_unicode(base64.b64decode(encoded_cmd).decode('utf-16-le')) + decoded_cmd = to_text(base64.b64decode(encoded_cmd).decode('utf-16-le')) display.vvv("EXEC %s" % decoded_cmd, host=self._winrm_host) else: display.vvv("EXEC %s" % cmd, host=self._winrm_host) @@ -300,9 +300,9 @@ class Connection(ConnectionBase): # FUTURE: determine buffer size at runtime via remote winrm config? def _put_file_stdin_iterator(self, in_path, out_path, buffer_size=250000): - in_size = os.path.getsize(to_bytes(in_path, errors='strict')) + in_size = os.path.getsize(to_bytes(in_path, errors='surrogate_or_strict')) offset = 0 - with open(to_bytes(in_path, errors='strict'), 'rb') as in_file: + with open(to_bytes(in_path, errors='surrogate_or_strict'), 'rb') as in_file: for out_data in iter((lambda:in_file.read(buffer_size)), ''): offset += len(out_data) self._display.vvvvv('WINRM PUT "%s" to "%s" (offset=%d size=%d)' % (in_path, out_path, offset, len(out_data)), host=self._winrm_host) @@ -318,7 +318,7 @@ class Connection(ConnectionBase): super(Connection, self).put_file(in_path, out_path) out_path = self._shell._unquote(out_path) display.vvv('PUT "%s" TO "%s"' % (in_path, out_path), host=self._winrm_host) - if not os.path.exists(to_bytes(in_path, errors='strict')): + if not os.path.exists(to_bytes(in_path, errors='surrogate_or_strict')): raise AnsibleFileNotFound('file or module does not exist: "%s"' % in_path) script_template = u''' @@ -357,7 +357,7 @@ class Connection(ConnectionBase): result = self._winrm_exec(cmd_parts[0], cmd_parts[1:], stdin_iterator=self._put_file_stdin_iterator(in_path, out_path)) # TODO: improve error handling if result.status_code != 0: - raise AnsibleError(to_str(result.std_err)) + raise AnsibleError(to_native(result.std_err)) put_output = json.loads(result.std_out) remote_sha1 = put_output.get("sha1") @@ -368,7 +368,7 @@ class Connection(ConnectionBase): local_sha1 = secure_hash(in_path) if not remote_sha1 == local_sha1: - raise AnsibleError("Remote sha1 hash {0} does not match local hash {1}".format(to_str(remote_sha1), to_str(local_sha1))) + raise AnsibleError("Remote sha1 hash {0} does not match local hash {1}".format(to_native(remote_sha1), to_native(local_sha1))) def fetch_file(self, in_path, out_path): super(Connection, self).fetch_file(in_path, out_path) @@ -407,7 +407,7 @@ class Connection(ConnectionBase): cmd_parts = self._shell._encode_script(script, as_list=True) result = self._winrm_exec(cmd_parts[0], cmd_parts[1:]) if result.status_code != 0: - raise IOError(to_str(result.std_err)) + raise IOError(to_native(result.std_err)) if result.std_out.strip() == '[DIR]': data = None else: @@ -418,9 +418,9 @@ class Connection(ConnectionBase): else: if not out_file: # If out_path is a directory and we're expecting a file, bail out now. - if os.path.isdir(to_bytes(out_path, errors='strict')): + if os.path.isdir(to_bytes(out_path, errors='surrogate_or_strict')): break - out_file = open(to_bytes(out_path, errors='strict'), 'wb') + out_file = open(to_bytes(out_path, errors='surrogate_or_strict'), 'wb') out_file.write(data) if len(data) < buffer_size: break diff --git a/lib/ansible/plugins/connection/zone.py b/lib/ansible/plugins/connection/zone.py index eaf0a049b9..3c1322d3ec 100644 --- a/lib/ansible/plugins/connection/zone.py +++ b/lib/ansible/plugins/connection/zone.py @@ -31,7 +31,8 @@ import traceback from ansible import constants as C from ansible.errors import AnsibleError from ansible.plugins.connection import ConnectionBase, BUFSIZE -from ansible.utils.unicode import to_bytes +from ansible.module_utils._text import to_bytes + try: from __main__ import display diff --git a/lib/ansible/plugins/filter/core.py b/lib/ansible/plugins/filter/core.py index 0889cee591..24c3901b80 100644 --- a/lib/ansible/plugins/filter/core.py +++ b/lib/ansible/plugins/filter/core.py @@ -19,7 +19,6 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type - import sys import base64 import itertools @@ -39,15 +38,6 @@ import uuid import yaml from jinja2.filters import environmentfilter -from ansible.compat.six import iteritems, string_types - -from ansible import errors -from ansible.parsing.yaml.dumper import AnsibleDumper -from ansible.utils.hashing import md5s, checksum_s -from ansible.utils.unicode import unicode_wrap, to_unicode -from ansible.utils.vars import merge_hash -from ansible.vars.hostvars import HostVars -from ansible.compat.six.moves import reduce try: import passlib.hash @@ -55,6 +45,16 @@ try: except: HAS_PASSLIB = False +from ansible import errors +from ansible.compat.six import iteritems, string_types +from ansible.compat.six.moves import reduce +from ansible.module_utils._text import to_text +from ansible.parsing.yaml.dumper import AnsibleDumper +from ansible.utils.hashing import md5s, checksum_s +from ansible.utils.unicode import unicode_wrap +from ansible.utils.vars import merge_hash +from ansible.vars.hostvars import HostVars + UUID_NAMESPACE_ANSIBLE = uuid.UUID('361E6D51-FAEC-444A-9079-341386DA8E2E') @@ -72,12 +72,12 @@ class AnsibleJSONEncoder(json.JSONEncoder): def to_yaml(a, *args, **kw): '''Make verbose, human readable yaml''' transformed = yaml.dump(a, Dumper=AnsibleDumper, allow_unicode=True, **kw) - return to_unicode(transformed) + return to_text(transformed) def to_nice_yaml(a, indent=4, *args, **kw): '''Make verbose, human readable yaml''' transformed = yaml.dump(a, Dumper=AnsibleDumper, indent=indent, allow_unicode=True, default_flow_style=False, **kw) - return to_unicode(transformed) + return to_text(transformed) def to_json(a, *args, **kw): ''' Convert the value to JSON ''' @@ -132,7 +132,7 @@ def fileglob(pathname): def regex_replace(value='', pattern='', replacement='', ignorecase=False): ''' Perform a `re.sub` returning a string ''' - value = to_unicode(value, errors='strict', nonstring='simplerepr') + value = to_text(value, errors='surrogate_or_strict', nonstring='simplerepr') if ignorecase: flags = re.I diff --git a/lib/ansible/plugins/lookup/__init__.py b/lib/ansible/plugins/lookup/__init__.py index 2a90321e40..7112b4ab0b 100644 --- a/lib/ansible/plugins/lookup/__init__.py +++ b/lib/ansible/plugins/lookup/__init__.py @@ -98,8 +98,8 @@ class LookupBase(with_metaclass(ABCMeta, object)): must be converted into python's unicode type as the strings will be run through jinja2 which has this requirement. You can use:: - from ansible.utils.unicode import to_unicode - result_string = to_unicode(result_string) + from ansible.module_utils._text import to_text + result_string = to_text(result_string) """ pass diff --git a/lib/ansible/plugins/lookup/csvfile.py b/lib/ansible/plugins/lookup/csvfile.py index 936288e26f..195a69e56a 100644 --- a/lib/ansible/plugins/lookup/csvfile.py +++ b/lib/ansible/plugins/lookup/csvfile.py @@ -22,7 +22,8 @@ import csv from ansible.errors import AnsibleError from ansible.plugins.lookup import LookupBase -from ansible.utils.unicode import to_bytes, to_str, to_unicode +from ansible.module_utils._text import to_bytes, to_native, to_text + class CSVRecoder: """ @@ -49,7 +50,7 @@ class CSVReader: def next(self): row = self.reader.next() - return [to_unicode(s) for s in row] + return [to_text(s) for s in row] def __iter__(self): return self @@ -66,7 +67,7 @@ class LookupModule(LookupBase): if row[0] == key: return row[int(col)] except Exception as e: - raise AnsibleError("csvfile: %s" % to_str(e)) + raise AnsibleError("csvfile: %s" % to_native(e)) return dflt diff --git a/lib/ansible/plugins/lookup/fileglob.py b/lib/ansible/plugins/lookup/fileglob.py index 66de4f1f45..3e0523fdd5 100644 --- a/lib/ansible/plugins/lookup/fileglob.py +++ b/lib/ansible/plugins/lookup/fileglob.py @@ -22,7 +22,8 @@ import glob from ansible.plugins.lookup import LookupBase from ansible.errors import AnsibleFileNotFound -from ansible.utils.unicode import to_bytes, to_unicode +from ansible.module_utils._text import to_bytes, to_text + class LookupModule(LookupBase): @@ -36,6 +37,6 @@ class LookupModule(LookupBase): except AnsibleFileNotFound: dwimmed_path = None if dwimmed_path: - globbed = glob.glob(to_bytes(os.path.join(dwimmed_path, term_file), errors='strict')) - ret.extend(to_unicode(g, errors='strict') for g in globbed if os.path.isfile(g)) + globbed = glob.glob(to_bytes(os.path.join(dwimmed_path, term_file), errors='surrogate_or_strict')) + ret.extend(to_text(g, errors='surrogate_or_strict') for g in globbed if os.path.isfile(g)) return ret diff --git a/lib/ansible/plugins/lookup/filetree.py b/lib/ansible/plugins/lookup/filetree.py index f4d96af876..7ff5621214 100644 --- a/lib/ansible/plugins/lookup/filetree.py +++ b/lib/ansible/plugins/lookup/filetree.py @@ -22,12 +22,6 @@ import pwd import grp import stat -from ansible.plugins.lookup import LookupBase -from ansible.utils.unicode import to_str - -from __main__ import display -warning = display.warning - HAVE_SELINUX=False try: import selinux @@ -35,6 +29,11 @@ try: except ImportError: pass +from ansible.plugins.lookup import LookupBase +from ansible.module_utils._text import to_native + +from __main__ import display + # If selinux fails to find a default, return an array of None def selinux_context(path): @@ -43,7 +42,7 @@ def selinux_context(path): try: # note: the selinux module uses byte strings on python2 and text # strings on python3 - ret = selinux.lgetfilecon_raw(to_str(path)) + ret = selinux.lgetfilecon_raw(to_native(path)) except OSError: return context if ret[0] != -1: @@ -60,7 +59,7 @@ def file_props(root, path): try: st = os.lstat(abspath) except OSError as e: - warning('filetree: Error using stat() on path %s (%s)' % (abspath, e)) + display.warning('filetree: Error using stat() on path %s (%s)' % (abspath, e)) return None ret = dict(root=root, path=path) @@ -74,7 +73,7 @@ def file_props(root, path): ret['state'] = 'file' ret['src'] = abspath else: - warning('filetree: Error file type of %s is not supported' % abspath) + display.warning('filetree: Error file type of %s is not supported' % abspath) return None ret['uid'] = st.st_uid diff --git a/lib/ansible/plugins/lookup/ini.py b/lib/ansible/plugins/lookup/ini.py index 5425a115f8..cbe0d13e6d 100644 --- a/lib/ansible/plugins/lookup/ini.py +++ b/lib/ansible/plugins/lookup/ini.py @@ -30,7 +30,7 @@ except ImportError: from ansible.errors import AnsibleError from ansible.plugins.lookup import LookupBase -from ansible.utils.unicode import to_bytes +from ansible.module_utils._text import to_bytes, to_text def _parse_params(term): @@ -59,13 +59,15 @@ class LookupModule(LookupBase): def read_properties(self, filename, key, dflt, is_regexp): config = StringIO() - config.write(u'[java_properties]\n' + open(to_bytes(filename, errors='strict')).read()) + current_cfg_file = open(to_bytes(filename, errors='surrogate_or_strict'), 'rb') + + config.write(u'[java_properties]\n' + to_text(current_cfg_file.read(), errors='surrogate_or_strict')) config.seek(0, os.SEEK_SET) self.cp.readfp(config) return self.get_value(key, 'java_properties', dflt, is_regexp) def read_ini(self, filename, key, section, dflt, is_regexp): - self.cp.readfp(open(to_bytes(filename, errors='strict'))) + self.cp.readfp(open(to_bytes(filename, errors='surrogate_or_strict'))) return self.get_value(key, section, dflt, is_regexp) def get_value(self, key, section, dflt, is_regexp): diff --git a/lib/ansible/plugins/lookup/shelvefile.py b/lib/ansible/plugins/lookup/shelvefile.py index 415bc358ed..e0ae4f9ad6 100644 --- a/lib/ansible/plugins/lookup/shelvefile.py +++ b/lib/ansible/plugins/lookup/shelvefile.py @@ -21,10 +21,10 @@ import shelve from ansible.errors import AnsibleError from ansible.plugins.lookup import LookupBase -from ansible.utils.unicode import to_bytes, to_unicode +from ansible.module_utils._text import to_bytes, to_text -class LookupModule(LookupBase): +class LookupModule(LookupBase): def read_shelve(self, shelve_filename, key): """ @@ -66,7 +66,7 @@ class LookupModule(LookupBase): if res is None: raise AnsibleError("Key %s not found in shelve file %s" % (key, file)) # Convert the value read to string - ret.append(to_unicode(res)) + ret.append(to_text(res)) break else: raise AnsibleError("Could not locate shelve file in lookup: %s" % file) diff --git a/lib/ansible/plugins/lookup/template.py b/lib/ansible/plugins/lookup/template.py index 2c722945d0..4ab0ad1ed0 100644 --- a/lib/ansible/plugins/lookup/template.py +++ b/lib/ansible/plugins/lookup/template.py @@ -21,7 +21,7 @@ import os from ansible.errors import AnsibleError from ansible.plugins.lookup import LookupBase -from ansible.utils.unicode import to_unicode, to_bytes +from ansible.module_utils._text import to_bytes, to_text try: from __main__ import display @@ -43,8 +43,8 @@ class LookupModule(LookupBase): lookupfile = self.find_file_in_search_path(variables, 'templates', term) display.vvvv("File lookup using %s as file" % lookupfile) if lookupfile: - with open(to_bytes(lookupfile, errors='strict'), 'r') as f: - template_data = to_unicode(f.read()) + with open(to_bytes(lookupfile, errors='surrogate_or_strict'), 'rb') as f: + template_data = to_text(f.read(), errors='surrogate_or_strict') # set jinja2 internal search path for includes if 'ansible_search_path' in variables: diff --git a/lib/ansible/plugins/lookup/url.py b/lib/ansible/plugins/lookup/url.py index be9d2e08b0..e84dbe0ae7 100644 --- a/lib/ansible/plugins/lookup/url.py +++ b/lib/ansible/plugins/lookup/url.py @@ -18,11 +18,11 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type +from ansible.compat.six.moves.urllib.error import HTTPError, URLError from ansible.errors import AnsibleError -from ansible.plugins.lookup import LookupBase +from ansible.module_utils._text import to_text from ansible.module_utils.urls import open_url, ConnectionError, SSLValidationError -from ansible.utils.unicode import to_unicode -from ansible.compat.six.moves.urllib.error import HTTPError, URLError +from ansible.plugins.lookup import LookupBase try: from __main__ import display @@ -52,5 +52,5 @@ class LookupModule(LookupBase): raise AnsibleError("Error connecting to %s: %s" % (term, str(e))) for line in response.read().splitlines(): - ret.append(to_unicode(line)) + ret.append(to_text(line)) return ret diff --git a/lib/ansible/plugins/shell/powershell.py b/lib/ansible/plugins/shell/powershell.py index 5cbaae74b3..b883dead22 100644 --- a/lib/ansible/plugins/shell/powershell.py +++ b/lib/ansible/plugins/shell/powershell.py @@ -22,9 +22,9 @@ import os import re import shlex -from ansible.compat.six import text_type from ansible.errors import AnsibleError -from ansible.utils.unicode import to_bytes, to_unicode +from ansible.module_utils._text import to_bytes, to_text + _common_args = ['PowerShell', '-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Unrestricted'] @@ -34,6 +34,7 @@ _powershell_version = os.environ.get('POWERSHELL_VERSION', None) if _powershell_version: _common_args = ['PowerShell', '-Version', _powershell_version] + _common_args[1:] + class ShellModule(object): # Common shell filenames that this plugin handles @@ -60,7 +61,7 @@ class ShellModule(object): raise AnsibleError("PowerShell environment value for key '%s' exceeds 32767 characters in length" % key) # powershell single quoted literals need single-quote doubling as their only escaping value = value.replace("'", "''") - return text_type(value) + return to_text(value, errors='surrogate_or_strict') def env_prefix(self, **kwargs): env = self.env.copy() @@ -164,7 +165,7 @@ class ShellModule(object): def build_module_command(self, env_string, shebang, cmd, arg_path=None, rm_tmp=None): cmd_parts = shlex.split(to_bytes(cmd), posix=False) - cmd_parts = map(to_unicode, cmd_parts) + cmd_parts = map(to_text, cmd_parts) if shebang and shebang.lower() == '#!powershell': if not self._unquote(cmd_parts[0]).lower().endswith('.ps1'): cmd_parts[0] = '"%s.ps1"' % self._unquote(cmd_parts[0]) @@ -219,7 +220,7 @@ class ShellModule(object): def _unquote(self, value): '''Remove any matching quotes that wrap the given value.''' - value = to_unicode(value or '') + value = to_text(value or '') m = re.match(r'^\s*?\'(.*?)\'\s*?$', value) if m: return m.group(1) @@ -244,7 +245,7 @@ class ShellModule(object): def _encode_script(self, script, as_list=False, strict_mode=True): '''Convert a PowerShell script to a single base64-encoded command.''' - script = to_unicode(script) + script = to_text(script) if strict_mode: script = u'Set-StrictMode -Version Latest\r\n%s' % script script = '\n'.join([x.strip() for x in script.splitlines() if x.strip()]) diff --git a/lib/ansible/plugins/strategy/__init__.py b/lib/ansible/plugins/strategy/__init__.py index d2d18558b1..34324f92b4 100644 --- a/lib/ansible/plugins/strategy/__init__.py +++ b/lib/ansible/plugins/strategy/__init__.py @@ -19,18 +19,11 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type -import json -import time -import zlib -from collections import defaultdict - from jinja2.exceptions import UndefinedError from ansible.compat.six.moves import queue as Queue -from ansible.compat.six import iteritems, text_type, string_types -from ansible import constants as C +from ansible.compat.six import iteritems, string_types from ansible.errors import AnsibleError, AnsibleParserError, AnsibleUndefinedVariable -from ansible.executor.play_iterator import PlayIterator from ansible.executor.task_result import TaskResult from ansible.inventory.host import Host from ansible.inventory.group import Group @@ -38,11 +31,10 @@ from ansible.playbook.helpers import load_list_of_blocks from ansible.playbook.included_file import IncludedFile from ansible.playbook.task_include import TaskInclude from ansible.playbook.role_include import IncludeRole -from ansible.plugins import action_loader, connection_loader, filter_loader, lookup_loader, module_loader, test_loader +from ansible.plugins import action_loader from ansible.template import Templar -from ansible.utils.unicode import to_unicode -from ansible.vars.unsafe_proxy import wrap_var from ansible.vars import combine_vars, strip_internal_keys +from ansible.module_utils._text import to_text try: @@ -138,7 +130,7 @@ class StrategyBase: ret_results = [] def get_original_host(host_name): - host_name = to_unicode(host_name) + host_name = to_text(host_name) if host_name in self._inventory._hosts_cache: return self._inventory._hosts_cache[host_name] else: @@ -161,7 +153,7 @@ class StrategyBase: target_handler_name = templar.template(handler_task.get_name()) if target_handler_name == handler_name: return handler_task - except (UndefinedError, AnsibleUndefinedVariable) as e: + except (UndefinedError, AnsibleUndefinedVariable): # We skip this handler due to the fact that it may be using # a variable in the name that was conditionally included via # set_fact or some other method, and we don't want to error @@ -182,7 +174,7 @@ class StrategyBase: target_handler_name = templar.template(target_handler.get_name()) if target_handler_name == handler_name: return True - except (UndefinedError, AnsibleUndefinedVariable) as e: + except (UndefinedError, AnsibleUndefinedVariable): pass return parent_handler_match(target_handler._parent, handler_name) else: @@ -567,14 +559,13 @@ class StrategyBase: # mark all of the hosts including this file as failed, send callbacks, # and increment the stats for this host for host in included_file._hosts: - tr = TaskResult(host=host, task=included_file._task, return_data=dict(failed=True, reason=to_unicode(e))) + tr = TaskResult(host=host, task=included_file._task, return_data=dict(failed=True, reason=to_text(e))) iterator.mark_host_failed(host) self._tqm._failed_hosts[host.name] = True self._tqm._stats.increment('failures', host.name) self._tqm.send_callback('v2_runner_on_failed', tr) return [] - # finally, send the callback and return the list of blocks loaded self._tqm.send_callback('v2_playbook_on_include', included_file) display.debug("done processing included file") diff --git a/lib/ansible/plugins/strategy/free.py b/lib/ansible/plugins/strategy/free.py index a3e54a66ef..7f92de60ac 100644 --- a/lib/ansible/plugins/strategy/free.py +++ b/lib/ansible/plugins/strategy/free.py @@ -26,7 +26,8 @@ from ansible.playbook.included_file import IncludedFile from ansible.plugins import action_loader from ansible.plugins.strategy import StrategyBase from ansible.template import Templar -from ansible.utils.unicode import to_unicode +from ansible.module_utils._text import to_text + try: from __main__ import display @@ -66,8 +67,7 @@ class StrategyModule(StrategyBase): break work_to_do = False # assume we have no more work to do - starting_host = last_host # save current position so we know when we've - # looped back around and need to break + starting_host = last_host # save current position so we know when we've looped back around and need to break # try and find an unblocked host with a task to run host_results = [] @@ -109,7 +109,7 @@ class StrategyModule(StrategyBase): display.debug("done getting variables") try: - task.name = to_unicode(templar.template(task.name, fail_on_undefined=False), nonstring='empty') + task.name = to_text(templar.template(task.name, fail_on_undefined=False), nonstring='empty') display.debug("done templating") except: # just ignore any errors during task name templating, @@ -120,10 +120,10 @@ class StrategyModule(StrategyBase): run_once = templar.template(task.run_once) or action and getattr(action, 'BYPASS_HOST_LOOP', False) if run_once: if action and getattr(action, 'BYPASS_HOST_LOOP', False): - raise AnsibleError("The '%s' module bypasses the host loop, which is currently not supported in the free strategy " \ + raise AnsibleError("The '%s' module bypasses the host loop, which is currently not supported in the free strategy " "and would instead execute for every host in the inventory list." % task.action, obj=task._ds) else: - display.warning("Using run_once with the free strategy is not currently supported. This task will still be " \ + display.warning("Using run_once with the free strategy is not currently supported. This task will still be " "executed for every host in the inventory list.") # check to see if this task should be skipped, due to it being a member of a @@ -143,7 +143,8 @@ class StrategyModule(StrategyBase): # handle step if needed, skip meta actions as they are used internally if not self._step or self._take_step(task, host_name): if task.any_errors_fatal: - display.warning("Using any_errors_fatal with the free strategy is not supported, as tasks are executed independently on each host") + display.warning("Using any_errors_fatal with the free strategy is not supported," + " as tasks are executed independently on each host") self._tqm.send_callback('v2_playbook_on_task_start', task, is_conditional=False) self._queue_task(host, task, task_vars, play_context) del task_vars diff --git a/lib/ansible/plugins/strategy/linear.py b/lib/ansible/plugins/strategy/linear.py index 50ba4a70eb..e5920200a1 100644 --- a/lib/ansible/plugins/strategy/linear.py +++ b/lib/ansible/plugins/strategy/linear.py @@ -29,7 +29,8 @@ from ansible.playbook.task import Task from ansible.plugins import action_loader from ansible.plugins.strategy import StrategyBase from ansible.template import Templar -from ansible.utils.unicode import to_unicode +from ansible.module_utils._text import to_text + try: from __main__ import display @@ -243,7 +244,7 @@ class StrategyModule(StrategyBase): saved_name = task.name display.debug("done copying, going to template now") try: - task.name = to_unicode(templar.template(task.name, fail_on_undefined=False), nonstring='empty') + task.name = to_text(templar.template(task.name, fail_on_undefined=False), nonstring='empty') display.debug("done templating") except: # just ignore any errors during task name templating, @@ -368,7 +369,7 @@ class StrategyModule(StrategyBase): for host in included_file._hosts: self._tqm._failed_hosts[host.name] = True iterator.mark_host_failed(host) - display.error(to_unicode(e), wrap_text=False) + display.error(to_text(e), wrap_text=False) include_failure = True continue diff --git a/lib/ansible/template/__init__.py b/lib/ansible/template/__init__.py index 05ec12f03d..2eec3bf3c7 100644 --- a/lib/ansible/template/__init__.py +++ b/lib/ansible/template/__init__.py @@ -25,8 +25,8 @@ import os import re from io import StringIO +from numbers import Number -from ansible.compat.six import string_types, text_type, binary_type from jinja2 import Environment from jinja2.loaders import FileSystemLoader from jinja2.exceptions import TemplateSyntaxError, UndefinedError @@ -34,19 +34,20 @@ from jinja2.utils import concat as j2_concat from jinja2.runtime import StrictUndefined from ansible import constants as C +from ansible.compat.six import string_types, text_type from ansible.errors import AnsibleError, AnsibleFilterError, AnsibleUndefinedVariable from ansible.plugins import filter_loader, lookup_loader, test_loader from ansible.template.safe_eval import safe_eval from ansible.template.template import AnsibleJ2Template from ansible.template.vars import AnsibleJ2Vars -from ansible.utils.unicode import to_unicode, to_str +from ansible.module_utils._text import to_native, to_text + try: from hashlib import sha1 except ImportError: from sha import sha as sha1 -from numbers import Number try: from __main__ import display @@ -107,6 +108,7 @@ def _escape_backslashes(data, jinja_env): return data + def _count_newlines_from_end(in_str): ''' Counts the number of newlines at the end of a string. This is used during @@ -255,10 +257,10 @@ class Templar: if prev_idx is not None: # replace the opening data.seek(prev_idx, os.SEEK_SET) - data.write(to_unicode(self.environment.comment_start_string)) + data.write(to_text(self.environment.comment_start_string)) # replace the closing data.seek(token_start, os.SEEK_SET) - data.write(to_unicode(self.environment.comment_end_string)) + data.write(to_text(self.environment.comment_end_string)) else: raise AnsibleError("Error while cleaning data for safety: unhandled regex match") @@ -293,7 +295,7 @@ class Templar: return self._clean_data(variable) else: # Do we need to convert these into text_type as well? - # return self._clean_data(to_unicode(variable._obj, nonstring='passthru')) + # return self._clean_data(to_text(variable._obj, nonstring='passthru')) return self._clean_data(variable._obj) try: @@ -330,7 +332,7 @@ class Templar: if convert_data and not self._no_type_regex.match(variable): # if this looks like a dictionary or list, convert it to such using the safe_eval method if (result.startswith("{") and not result.startswith(self.environment.variable_start_string)) or \ - result.startswith("[") or result in ("True", "False"): + result.startswith("[") or result in ("True", "False"): eval_results = safe_eval(result, locals=self._available_variables, include_exceptions=True) if eval_results[1] is None: result = eval_results[0] @@ -383,7 +385,7 @@ class Templar: returns True if the data contains a variable pattern ''' if isinstance(data, string_types): - for marker in [self.environment.block_start_string, self.environment.variable_start_string, self.environment.comment_start_string]: + for marker in (self.environment.block_start_string, self.environment.variable_start_string, self.environment.comment_start_string): if marker in data: return True return False @@ -399,8 +401,9 @@ class Templar: first_part = variable.split("|")[0].split(".")[0].split("[")[0] if (contains_filters or first_part in self._available_variables) and self.environment.variable_start_string not in variable: if bare_deprecated: - display.deprecated("Using bare variables is deprecated. Update your playbooks so that the environment value uses the full variable syntax ('%s%s%s')" % - (self.environment.variable_start_string, variable, self.environment.variable_end_string)) + display.deprecated("Using bare variables is deprecated." + " Update your playbooks so that the environment value uses the full variable syntax ('%s%s%s')" % + (self.environment.variable_start_string, variable, self.environment.variable_end_string)) return "%s%s%s" % (self.environment.variable_start_string, variable, self.environment.variable_end_string) # the variable didn't meet the conditions to be converted, @@ -485,10 +488,10 @@ class Templar: try: t = myenv.from_string(data) except TemplateSyntaxError as e: - raise AnsibleError("template error while templating string: %s. String: %s" % (to_str(e), to_str(data))) + raise AnsibleError("template error while templating string: %s. String: %s" % (to_native(e), to_native(data))) except Exception as e: - if 'recursion' in to_str(e): - raise AnsibleError("recursive loop detected in template string: %s" % to_str(data)) + if 'recursion' in to_native(e): + raise AnsibleError("recursive loop detected in template string: %s" % to_native(data)) else: return data @@ -503,13 +506,13 @@ class Templar: try: res = j2_concat(rf) except TypeError as te: - if 'StrictUndefined' in to_str(te): - errmsg = "Unable to look up a name or access an attribute in template string (%s).\n" % to_str(data) - errmsg += "Make sure your variable name does not contain invalid characters like '-': %s" % to_str(te) + if 'StrictUndefined' in to_native(te): + errmsg = "Unable to look up a name or access an attribute in template string (%s).\n" % to_native(data) + errmsg += "Make sure your variable name does not contain invalid characters like '-': %s" % to_native(te) raise AnsibleUndefinedVariable(errmsg) else: - display.debug("failing because of a type error, template data is: %s" % to_str(data)) - raise AnsibleError("Unexpected templating type error occurred on (%s): %s" % (to_str(data),to_str(te))) + display.debug("failing because of a type error, template data is: %s" % to_native(data)) + raise AnsibleError("Unexpected templating type error occurred on (%s): %s" % (to_native(data),to_native(te))) if preserve_trailing_newlines: # The low level calls above do not preserve the newline @@ -534,4 +537,3 @@ class Templar: else: #TODO: return warning about undefined var return data - diff --git a/lib/ansible/template/vars.py b/lib/ansible/template/vars.py index badf93b1e8..efb1ce734f 100644 --- a/lib/ansible/template/vars.py +++ b/lib/ansible/template/vars.py @@ -21,7 +21,8 @@ __metaclass__ = type from ansible.compat.six import iteritems from jinja2.utils import missing -from ansible.utils.unicode import to_unicode +from ansible.module_utils._text import to_native + __all__ = ['AnsibleJ2Vars'] @@ -88,7 +89,7 @@ class AnsibleJ2Vars: try: value = self._templar.template(variable) except Exception as e: - raise type(e)(to_unicode(variable) + ': ' + e.message) + raise type(e)(to_native(variable) + ': ' + e.message) return value def add_locals(self, locals): @@ -99,4 +100,3 @@ class AnsibleJ2Vars: if locals is None: return self return AnsibleJ2Vars(self._templar, self._globals, locals=locals, *self._extras) - diff --git a/lib/ansible/utils/display.py b/lib/ansible/utils/display.py index 76ecb22715..8034f73ee3 100644 --- a/lib/ansible/utils/display.py +++ b/lib/ansible/utils/display.py @@ -35,7 +35,8 @@ from termios import TIOCGWINSZ from ansible import constants as C from ansible.errors import AnsibleError from ansible.utils.color import stringc -from ansible.utils.unicode import to_bytes, to_unicode +from ansible.module_utils._text import to_bytes, to_text + try: # Python 2 @@ -45,7 +46,6 @@ except NameError: pass - logger = None #TODO: make this a logging callback instead if C.DEFAULT_LOG_PATH: @@ -105,7 +105,7 @@ class Display: """ Display a message to the user Note: msg *must* be a unicode string to prevent UnicodeError tracebacks. - """ + """ # FIXME: this needs to be implemented #msg = utils.sanitize_output(msg) @@ -124,7 +124,7 @@ class Display: # Convert back to text string on python3 # We first convert to a byte string so that we get rid of # characters that are invalid in the user's locale - msg2 = to_unicode(msg2, self._output_encoding(stderr=stderr)) + msg2 = to_text(msg2, self._output_encoding(stderr=stderr)) if not stderr: fileobj = sys.stdout @@ -149,7 +149,7 @@ class Display: # Convert back to text string on python3 # We first convert to a byte string so that we get rid of # characters that are invalid in the user's locale - msg2 = to_unicode(msg2, self._output_encoding(stderr=stderr)) + msg2 = to_text(msg2, self._output_encoding(stderr=stderr)) if color == C.COLOR_ERROR: logger.error(msg2) @@ -279,7 +279,7 @@ class Display: if sys.version_info >= (3,): # Convert back into text on python3. We do this double conversion # to get rid of characters that are illegal in the user's locale - prompt_string = to_unicode(prompt_string) + prompt_string = to_text(prompt_string) if private: return getpass.getpass(msg) @@ -323,7 +323,7 @@ class Display: result = do_encrypt(result, encrypt, salt_size, salt) # handle utf-8 chars - result = to_unicode(result, errors='strict') + result = to_text(result, errors='surrogate_or_strict') return result @staticmethod diff --git a/lib/ansible/utils/hashing.py b/lib/ansible/utils/hashing.py index a486f3e7b0..3c5b125b0b 100644 --- a/lib/ansible/utils/hashing.py +++ b/lib/ansible/utils/hashing.py @@ -39,14 +39,14 @@ except ImportError: _md5 = None from ansible.errors import AnsibleError -from ansible.utils.unicode import to_bytes +from ansible.module_utils._text import to_bytes def secure_hash_s(data, hash_func=sha1): ''' Return a secure hash hex digest of data. ''' digest = hash_func() - data = to_bytes(data, errors='strict') + data = to_bytes(data, errors='surrogate_or_strict') digest.update(data) return digest.hexdigest() @@ -54,12 +54,12 @@ def secure_hash_s(data, hash_func=sha1): def secure_hash(filename, hash_func=sha1): ''' Return a secure hash hex digest of local file, None if file is not present or a directory. ''' - if not os.path.exists(to_bytes(filename, errors='strict')) or os.path.isdir(to_bytes(filename, errors='strict')): + if not os.path.exists(to_bytes(filename, errors='surrogate_or_strict')) or os.path.isdir(to_bytes(filename, errors='strict')): return None digest = hash_func() blocksize = 64 * 1024 try: - infile = open(to_bytes(filename, errors='strict'), 'rb') + infile = open(to_bytes(filename, errors='surrogate_or_strict'), 'rb') block = infile.read(blocksize) while block: digest.update(block) diff --git a/lib/ansible/utils/path.py b/lib/ansible/utils/path.py index 989356a79a..0820b7f993 100644 --- a/lib/ansible/utils/path.py +++ b/lib/ansible/utils/path.py @@ -20,11 +20,12 @@ __metaclass__ = type import os from errno import EEXIST from ansible.errors import AnsibleError -from ansible.utils.unicode import to_bytes, to_str, to_unicode -from ansible.compat.six import PY2 +from ansible.module_utils._text import to_bytes, to_native, to_text + __all__ = ['unfrackpath', 'makedirs_safe'] + def unfrackpath(path): ''' Returns a path that is free of symlinks, environment @@ -40,10 +41,9 @@ def unfrackpath(path): example:: '$HOME/../../var/mail' becomes '/var/spool/mail' ''' - canonical_path = os.path.normpath(os.path.realpath(os.path.expanduser(os.path.expandvars(to_bytes(path, errors='strict'))))) - if PY2: - return to_unicode(canonical_path, errors='strict') - return to_unicode(canonical_path, errors='surrogateescape') + canonical_path = os.path.normpath(os.path.realpath(os.path.expanduser(os.path.expandvars(to_bytes(path, errors='surrogate_or_strict'))))) + return to_text(canonical_path, errors='surrogate_or_strict') + def makedirs_safe(path, mode=None): '''Safe way to create dirs in muliprocess/thread environments. @@ -64,4 +64,4 @@ def makedirs_safe(path, mode=None): os.makedirs(b_rpath) except OSError as e: if e.errno != EEXIST: - raise AnsibleError("Unable to create local directories(%s): %s" % (to_str(rpath), to_str(e))) + raise AnsibleError("Unable to create local directories(%s): %s" % (to_native(rpath), to_native(e))) diff --git a/lib/ansible/utils/shlex.py b/lib/ansible/utils/shlex.py index 2dee15816b..1da7c66e58 100644 --- a/lib/ansible/utils/shlex.py +++ b/lib/ansible/utils/shlex.py @@ -21,8 +21,7 @@ __metaclass__ = type import shlex from ansible.compat.six import PY3 - -from ansible.utils.unicode import to_bytes, to_unicode +from ansible.module_utils._text import to_bytes, to_text if PY3: @@ -31,5 +30,5 @@ if PY3: else: # shlex.split() wants bytes (i.e. ``str``) input on Python 2 def shlex_split(s, comments=False, posix=True): - return map(to_unicode, shlex.split(to_bytes(s), comments, posix)) + return map(to_text, shlex.split(to_bytes(s), comments, posix)) shlex_split.__doc__ = shlex.split.__doc__ diff --git a/lib/ansible/utils/unicode.py b/lib/ansible/utils/unicode.py index bc184ca923..067877c0a7 100644 --- a/lib/ansible/utils/unicode.py +++ b/lib/ansible/utils/unicode.py @@ -19,264 +19,48 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type -from ansible.compat.six import string_types, text_type, binary_type, PY3 +from ansible.module_utils._text import to_bytes as _to_bytes, to_text, to_native -# to_bytes and to_unicode were written by Toshio Kuratomi for the -# python-kitchen library https://pypi.python.org/pypi/kitchen -# They are licensed in kitchen under the terms of the GPLv2+ -# They were copied and modified for use in ansible by Toshio in Jan 2015 -# (simply removing the deprecated features) +try: + from __main__ import display +except ImportError: + from ansible.utils.display import Display + display = Display() -#: Aliases for the utf-8 codec -_UTF8_ALIASES = frozenset(('utf-8', 'UTF-8', 'utf8', 'UTF8', 'utf_8', 'UTF_8', - 'utf', 'UTF', 'u8', 'U8')) -#: Aliases for the latin-1 codec -_LATIN1_ALIASES = frozenset(('latin-1', 'LATIN-1', 'latin1', 'LATIN1', - 'latin', 'LATIN', 'l1', 'L1', 'cp819', 'CP819', '8859', 'iso8859-1', - 'ISO8859-1', 'iso-8859-1', 'ISO-8859-1')) -# EXCEPTION_CONVERTERS is defined below due to using to_unicode +__all__ = ('to_bytes', 'to_unicode', 'to_str', 'unicode_wrap') -def to_unicode(obj, encoding='utf-8', errors='replace', nonstring=None): - '''Convert an object into a :class:`unicode` string - :arg obj: Object to convert to a :class:`unicode` string. This should - normally be a byte :class:`str` - :kwarg encoding: What encoding to try converting the byte :class:`str` as. - Defaults to :term:`utf-8` - :kwarg errors: If errors are found while decoding, perform this action. - Defaults to ``replace`` which replaces the invalid bytes with - a character that means the bytes were unable to be decoded. Other - values are the same as the error handling schemes in the `codec base - classes - <http://docs.python.org/library/codecs.html#codec-base-classes>`_. - For instance ``strict`` which raises an exception and ``ignore`` which - simply omits the non-decodable characters. - :kwarg nonstring: How to treat nonstring values. Possible values are: +### +### Backwards compat +### - :simplerepr: Attempt to call the object's "simple representation" - method and return that value. Python-2.3+ has two methods that - try to return a simple representation: :meth:`object.__unicode__` - and :meth:`object.__str__`. We first try to get a usable value - from :meth:`object.__unicode__`. If that fails we try the same - with :meth:`object.__str__`. - :empty: Return an empty :class:`unicode` string - :strict: Raise a :exc:`TypeError` - :passthru: Return the object unchanged - :repr: Attempt to return a :class:`unicode` string of the repr of the - object +def to_bytes(*args, **kwargs): + display.deprecated(u'ansible.utils.unicode.to_bytes is deprecated. Use ansible.module_utils._text.to_bytes instead', version=u'2.4') + if 'errors' not in kwargs: + kwargs['errors'] = 'replace' + return _to_bytes(*args, **kwargs) - Default is ``simplerepr`` - :raises TypeError: if :attr:`nonstring` is ``strict`` and - a non-:class:`basestring` object is passed in or if :attr:`nonstring` - is set to an unknown value - :raises UnicodeDecodeError: if :attr:`errors` is ``strict`` and - :attr:`obj` is not decodable using the given encoding - :returns: :class:`unicode` string or the original object depending on the - value of :attr:`nonstring`. +def to_unicode(*args, **kwargs): + display.deprecated(u'ansible.utils.unicode.to_unicode is deprecated. Use ansible.module_utils._text.to_text instead', version=u'2.4') + if 'errors' not in kwargs: + kwargs['errors'] = 'replace' + return to_text(*args, **kwargs) - Usually this should be used on a byte :class:`str` but it can take both - byte :class:`str` and :class:`unicode` strings intelligently. Nonstring - objects are handled in different ways depending on the setting of the - :attr:`nonstring` parameter. - The default values of this function are set so as to always return - a :class:`unicode` string and never raise an error when converting from - a byte :class:`str` to a :class:`unicode` string. However, when you do - not pass validly encoded text (or a nonstring object), you may end up with - output that you don't expect. Be sure you understand the requirements of - your data, not just ignore errors by passing it through this function. - ''' - # Could use isbasestring/isunicode here but we want this code to be as - # fast as possible - if isinstance(obj, text_type): - return obj - if isinstance(obj, binary_type): - if encoding in _UTF8_ALIASES: - return text_type(obj, 'utf-8', errors) - if encoding in _LATIN1_ALIASES: - return text_type(obj, 'latin-1', errors) - return obj.decode(encoding, errors) +def to_str(*args, **kwargs): + display.deprecated(u'ansible.utils.unicode.to_str is deprecated. Use ansible.module_utils._text.to_native instead', version=u'2.4') + if 'errors' not in kwargs: + kwargs['errors'] = 'replace' + return to_native(*args, **kwargs) - if not nonstring: - nonstring = 'simplerepr' - if nonstring == 'empty': - return u'' - elif nonstring == 'passthru': - return obj - elif nonstring == 'simplerepr': - try: - simple = obj.__unicode__() - except (AttributeError, UnicodeError): - simple = None - if not simple: - try: - simple = text_type(obj) - except UnicodeError: - try: - simple = obj.__str__() - except (UnicodeError, AttributeError): - simple = u'' - if isinstance(simple, binary_type): - return text_type(simple, encoding, errors) - return simple - elif nonstring in ('repr', 'strict'): - obj_repr = repr(obj) - if isinstance(obj_repr, binary_type): - obj_repr = text_type(obj_repr, encoding, errors) - if nonstring == 'repr': - return obj_repr - raise TypeError('to_unicode was given "%(obj)s" which is neither' - ' a byte string (str) or a unicode string' % - {'obj': obj_repr.encode(encoding, 'replace')}) +### End Backwards compat - raise TypeError('nonstring value, %(param)s, is not set to a valid' - ' action' % {'param': nonstring}) -def to_bytes(obj, encoding='utf-8', errors='replace', nonstring=None): - '''Convert an object into a byte :class:`str` - - :arg obj: Object to convert to a byte :class:`str`. This should normally - be a :class:`unicode` string. - :kwarg encoding: Encoding to use to convert the :class:`unicode` string - into a byte :class:`str`. Defaults to :term:`utf-8`. - :kwarg errors: If errors are found while encoding, perform this action. - Defaults to ``replace`` which replaces the invalid bytes with - a character that means the bytes were unable to be encoded. Other - values are the same as the error handling schemes in the `codec base - classes - <http://docs.python.org/library/codecs.html#codec-base-classes>`_. - For instance ``strict`` which raises an exception and ``ignore`` which - simply omits the non-encodable characters. - :kwarg nonstring: How to treat nonstring values. Possible values are: - - :simplerepr: Attempt to call the object's "simple representation" - method and return that value. Python-2.3+ has two methods that - try to return a simple representation: :meth:`object.__unicode__` - and :meth:`object.__str__`. We first try to get a usable value - from :meth:`object.__str__`. If that fails we try the same - with :meth:`object.__unicode__`. - :empty: Return an empty byte :class:`str` - :strict: Raise a :exc:`TypeError` - :passthru: Return the object unchanged - :repr: Attempt to return a byte :class:`str` of the :func:`repr` of the - object - - Default is ``simplerepr``. - - :raises TypeError: if :attr:`nonstring` is ``strict`` and - a non-:class:`basestring` object is passed in or if :attr:`nonstring` - is set to an unknown value. - :raises UnicodeEncodeError: if :attr:`errors` is ``strict`` and all of the - bytes of :attr:`obj` are unable to be encoded using :attr:`encoding`. - :returns: byte :class:`str` or the original object depending on the value - of :attr:`nonstring`. - - .. warning:: - - If you pass a byte :class:`str` into this function the byte - :class:`str` is returned unmodified. It is **not** re-encoded with - the specified :attr:`encoding`. The easiest way to achieve that is:: - - to_bytes(to_unicode(text), encoding='utf-8') - - The initial :func:`to_unicode` call will ensure text is - a :class:`unicode` string. Then, :func:`to_bytes` will turn that into - a byte :class:`str` with the specified encoding. - - Usually, this should be used on a :class:`unicode` string but it can take - either a byte :class:`str` or a :class:`unicode` string intelligently. - Nonstring objects are handled in different ways depending on the setting - of the :attr:`nonstring` parameter. - - The default values of this function are set so as to always return a byte - :class:`str` and never raise an error when converting from unicode to - bytes. However, when you do not pass an encoding that can validly encode - the object (or a non-string object), you may end up with output that you - don't expect. Be sure you understand the requirements of your data, not - just ignore errors by passing it through this function. - ''' - # Could use isbasestring, isbytestring here but we want this to be as fast - # as possible - if isinstance(obj, binary_type): - return obj - if isinstance(obj, text_type): - return obj.encode(encoding, errors) - if not nonstring: - nonstring = 'simplerepr' - - if nonstring == 'empty': - return b'' - elif nonstring == 'passthru': - return obj - elif nonstring == 'simplerepr': - try: - simple = str(obj) - except UnicodeError: - try: - simple = obj.__str__() - except (AttributeError, UnicodeError): - simple = None - if not simple: - try: - simple = obj.__unicode__() - except (AttributeError, UnicodeError): - simple = b'' - if isinstance(simple, text_type): - simple = simple.encode(encoding, 'replace') - return simple - elif nonstring in ('repr', 'strict'): - try: - obj_repr = obj.__repr__() - except (AttributeError, UnicodeError): - obj_repr = b'' - if isinstance(obj_repr, text_type): - obj_repr = obj_repr.encode(encoding, errors) - else: - obj_repr = binary_type(obj_repr) - if nonstring == 'repr': - return obj_repr - raise TypeError('to_bytes was given "%(obj)s" which is neither' - ' a unicode string or a byte string (str)' % {'obj': obj_repr}) - - raise TypeError('nonstring value, %(param)s, is not set to a valid' - ' action' % {'param': nonstring}) - - -# force the return value of a function to be unicode. Use with partial to -# ensure that a filter will return unicode values. def unicode_wrap(func, *args, **kwargs): - return to_unicode(func(*args, **kwargs), nonstring='passthru') - + """If a function returns a string, force it to be a text string. -# Alias for converting to native strings. -# Native strings are the default string type for the particular version of -# python. The objects are called "str" in both py2 and py3 but they mean -# different things. In py2, it's a byte string like in C. In py3 it's an -# abstract text type (like py2's unicode type). -# -# Use this when raising exceptions and wanting to get the string -# representation of an object for the exception message. For example: -# -# try: -# do_something() -# except Exception as e: -# raise AnsibleError(to_str(e)) -# -# Note that this is because python's exception handling expects native strings -# and doe the wrong thing if given the other sort of string (in py2, if given -# unicode strings, it could traceback or omit the message. in py3, if given -# byte strings it prints their repr (so the message ends up as b'message'). -# -# If you use ansible's API instead of re-raising an exception, use to_unicode -# instead: -# -# try: -# do_something() -# except Exception as e: -# display.warn(to_unicode(e)) -if PY3: - to_str = to_unicode -else: - to_str = to_bytes + Use with partial to ensure that filter plugins will return text values. + """ + return to_text(func(*args, **kwargs), nonstring='passthru') diff --git a/lib/ansible/utils/vars.py b/lib/ansible/utils/vars.py index ee3dd5feaa..40ad17109f 100644 --- a/lib/ansible/utils/vars.py +++ b/lib/ansible/utils/vars.py @@ -28,7 +28,7 @@ from ansible.compat.six import iteritems, string_types from ansible import constants as C from ansible.errors import AnsibleError from ansible.parsing.splitter import parse_kv -from ansible.utils.unicode import to_unicode, to_str +from ansible.module_utils._text import to_native, to_text def _validate_mutable_mappings(a, b): @@ -49,11 +49,12 @@ def _validate_mutable_mappings(a, b): try: myvars.append(dumps(x)) except: - myvars.append(to_str(x)) + myvars.append(to_native(x)) raise AnsibleError("failed to combine variables, expected dicts but got a '{0}' and a '{1}': \n{2}\n{3}".format( a.__class__.__name__, b.__class__.__name__, myvars[0], myvars[1]) ) + def combine_vars(a, b): """ Return a copy of dictionaries of variables based on configured hash behavior @@ -68,6 +69,7 @@ def combine_vars(a, b): result.update(b) return result + def merge_hash(a, b): """ Recursively merges hash b into a so that keys from b take precedence over keys from a @@ -95,10 +97,11 @@ def merge_hash(a, b): return result + def load_extra_vars(loader, options): extra_vars = {} for extra_vars_opt in options.extra_vars: - extra_vars_opt = to_unicode(extra_vars_opt, errors='strict') + extra_vars_opt = to_text(extra_vars_opt, errors='surrogate_or_strict') if extra_vars_opt.startswith(u"@"): # Argument is a YAML file (JSON is a subset of YAML) data = loader.load_from_file(extra_vars_opt[1:]) @@ -111,6 +114,7 @@ def load_extra_vars(loader, options): extra_vars = combine_vars(extra_vars, data) return extra_vars + def load_options_vars(options): options_vars = {} # For now only return check mode, but we can easily return more @@ -118,6 +122,7 @@ def load_options_vars(options): options_vars['ansible_check_mode'] = options.check return options_vars + def isidentifier(ident): """ Determines, if string is valid Python identifier using the ast module. @@ -148,4 +153,3 @@ def isidentifier(ident): return False return True - diff --git a/lib/ansible/vars/unsafe_proxy.py b/lib/ansible/vars/unsafe_proxy.py index ac5cce24af..4238d4d8e5 100644 --- a/lib/ansible/vars/unsafe_proxy.py +++ b/lib/ansible/vars/unsafe_proxy.py @@ -54,18 +54,21 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type import json - -from ansible.utils.unicode import to_unicode from ansible.compat.six import string_types, text_type +from ansible.module_utils._text import to_text + __all__ = ['UnsafeProxy', 'AnsibleUnsafe', 'AnsibleJSONUnsafeEncoder', 'AnsibleJSONUnsafeDecoder', 'wrap_var'] + class AnsibleUnsafe(object): __UNSAFE__ = True + class AnsibleUnsafeText(text_type, AnsibleUnsafe): pass + class UnsafeProxy(object): def __new__(cls, obj, *args, **kwargs): # In our usage we should only receive unicode strings. @@ -73,10 +76,11 @@ class UnsafeProxy(object): # we're given but we may want to take it out for testing and sanitize # our input instead. if isinstance(obj, string_types): - obj = to_unicode(obj, errors='strict') + obj = to_text(obj, errors='surrogate_or_strict') return AnsibleUnsafeText(obj) return obj + class AnsibleJSONUnsafeEncoder(json.JSONEncoder): def encode(self, obj): if isinstance(obj, AnsibleUnsafe): @@ -84,6 +88,7 @@ class AnsibleJSONUnsafeEncoder(json.JSONEncoder): else: return super(AnsibleJSONUnsafeEncoder, self).encode(obj) + class AnsibleJSONUnsafeDecoder(json.JSONDecoder): def decode(self, obj): value = super(AnsibleJSONUnsafeDecoder, self).decode(obj) @@ -92,6 +97,7 @@ class AnsibleJSONUnsafeDecoder(json.JSONDecoder): else: return value + def _wrap_dict(v): for k in v.keys(): if v[k] is not None: @@ -115,4 +121,3 @@ def wrap_var(v): if v is not None and not isinstance(v, AnsibleUnsafe): v = UnsafeProxy(v) return v - diff --git a/test/units/mock/procenv.py b/test/units/mock/procenv.py index abf436b94d..e9d470c079 100644 --- a/test/units/mock/procenv.py +++ b/test/units/mock/procenv.py @@ -27,7 +27,8 @@ from contextlib import contextmanager from io import BytesIO, StringIO from ansible.compat.six import PY3 from ansible.compat.tests import unittest -from ansible.utils.unicode import to_bytes +from ansible.module_utils._text import to_bytes + @contextmanager def swap_stdin_and_argv(stdin_data='', argv_data=tuple()): @@ -48,6 +49,7 @@ def swap_stdin_and_argv(stdin_data='', argv_data=tuple()): sys.stdin = real_stdin sys.argv = real_argv + @contextmanager def swap_stdout(): """ @@ -62,6 +64,7 @@ def swap_stdout(): yield fake_stream sys.stdout = old_stdout + class ModuleTestCase(unittest.TestCase): def setUp(self, module_args=None): if module_args is None: diff --git a/test/units/parsing/vault/test_vault.py b/test/units/parsing/vault/test_vault.py index 4fe0fff5e0..1be56413b8 100644 --- a/test/units/parsing/vault/test_vault.py +++ b/test/units/parsing/vault/test_vault.py @@ -30,11 +30,12 @@ from binascii import hexlify from nose.plugins.skip import SkipTest from ansible.compat.tests import unittest -from ansible.utils.unicode import to_bytes, to_unicode from ansible import errors from ansible.parsing.vault import VaultLib from ansible.parsing import vault +from ansible.module_utils._text import to_bytes + # Counter import fails for 2.0.1, requires >= 2.6.1 from pip try: diff --git a/test/units/parsing/vault/test_vault_editor.py b/test/units/parsing/vault/test_vault_editor.py index 5ef6be4daf..3b7ccf2582 100644 --- a/test/units/parsing/vault/test_vault_editor.py +++ b/test/units/parsing/vault/test_vault_editor.py @@ -27,11 +27,12 @@ from nose.plugins.skip import SkipTest from ansible.compat.tests import unittest from ansible.compat.tests.mock import patch -from ansible.utils.unicode import to_bytes, to_unicode from ansible import errors from ansible.parsing.vault import VaultLib from ansible.parsing.vault import VaultEditor +from ansible.module_utils._text import to_bytes, to_text + # Counter import fails for 2.0.1, requires >= 2.6.1 from pip try: @@ -66,6 +67,7 @@ v11_data = """$ANSIBLE_VAULT;1.1;AES256 3631633031323837340a396530313963373030343933616133393566366137363761373930663833 3739""" + class TestVaultEditor(unittest.TestCase): def setUp(self): @@ -121,20 +123,19 @@ class TestVaultEditor(unittest.TestCase): error_hit = False try: ve.decrypt_file(v10_file.name) - except errors.AnsibleError as e: + except errors.AnsibleError: error_hit = True # verify decrypted content f = open(v10_file.name, "rb") - fdata = to_unicode(f.read()) + fdata = to_text(f.read()) f.close() os.unlink(v10_file.name) - assert error_hit == False, "error decrypting 1.0 file" + assert error_hit is False, "error decrypting 1.0 file" assert fdata.strip() == "foo", "incorrect decryption of 1.0 file: %s" % fdata.strip() - def test_decrypt_1_1(self): if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2: raise SkipTest @@ -149,20 +150,19 @@ class TestVaultEditor(unittest.TestCase): error_hit = False try: ve.decrypt_file(v11_file.name) - except errors.AnsibleError as e: + except errors.AnsibleError: error_hit = True # verify decrypted content f = open(v11_file.name, "rb") - fdata = to_unicode(f.read()) + fdata = to_text(f.read()) f.close() os.unlink(v11_file.name) - assert error_hit == False, "error decrypting 1.0 file" + assert error_hit is False, "error decrypting 1.0 file" assert fdata.strip() == "foo", "incorrect decryption of 1.0 file: %s" % fdata.strip() - @unittest.skipIf(sys.version_info[0] >= 3, "VaultAES still needs to be ported to Python 3") def test_rekey_migration(self): """ @@ -182,7 +182,7 @@ class TestVaultEditor(unittest.TestCase): error_hit = False try: ve.rekey_file(v10_file.name, 'ansible2') - except errors.AnsibleError as e: + except errors.AnsibleError: error_hit = True # verify decrypted content @@ -190,7 +190,7 @@ class TestVaultEditor(unittest.TestCase): fdata = f.read() f.close() - assert error_hit == False, "error rekeying 1.0 file to 1.1" + assert error_hit is False, "error rekeying 1.0 file to 1.1" # ensure filedata can be decrypted, is 1.1 and is AES256 vl = VaultLib("ansible2") @@ -198,13 +198,11 @@ class TestVaultEditor(unittest.TestCase): error_hit = False try: dec_data = vl.decrypt(fdata) - except errors.AnsibleError as e: + except errors.AnsibleError: error_hit = True os.unlink(v10_file.name) assert vl.cipher_name == "AES256", "wrong cipher name set after rekey: %s" % vl.cipher_name - assert error_hit == False, "error decrypting migrated 1.0 file" + assert error_hit is False, "error decrypting migrated 1.0 file" assert dec_data.strip() == "foo", "incorrect decryption of rekeyed/migrated file: %s" % dec_data - - diff --git a/test/units/parsing/yaml/test_loader.py b/test/units/parsing/yaml/test_loader.py index 1e224f6704..2b166f2bd4 100644 --- a/test/units/parsing/yaml/test_loader.py +++ b/test/units/parsing/yaml/test_loader.py @@ -32,7 +32,6 @@ from ansible.parsing.yaml.loader import AnsibleLoader from ansible.parsing import vault from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode from ansible.parsing.yaml.dumper import AnsibleDumper -from ansible.utils.unicode import to_bytes from units.mock.yaml_helper import YamlTestUtils @@ -41,6 +40,7 @@ try: except ImportError: from yaml.parser import ParserError + class NameStringIO(StringIO): """In py2.6, StringIO doesn't let you set name because a baseclass has it as readonly property""" @@ -49,6 +49,7 @@ class NameStringIO(StringIO): def __init__(self, *args, **kwargs): super(NameStringIO, self).__init__(*args, **kwargs) + class TestAnsibleLoaderBasic(unittest.TestCase): def setUp(self): @@ -283,6 +284,7 @@ class TestAnsibleLoaderVault(unittest.TestCase, YamlTestUtils): self.assertFalse(plaintext_var != vault_string) self.assertFalse(vault_string != plaintext_var) + class TestAnsibleLoaderPlay(unittest.TestCase): def setUp(self): diff --git a/test/units/plugins/action/test_action.py b/test/units/plugins/action/test_action.py index 9a4cb74f3e..d9b2d33864 100644 --- a/test/units/plugins/action/test_action.py +++ b/test/units/plugins/action/test_action.py @@ -21,8 +21,6 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type -import ast -import json import pipes import os @@ -33,7 +31,6 @@ except ImportError: from nose.tools import eq_, raises -from ansible.release import __version__ as ansible_version from ansible import constants as C from ansible.compat.six import text_type from ansible.compat.tests import unittest @@ -41,12 +38,12 @@ from ansible.compat.tests.mock import patch, MagicMock, mock_open from ansible.errors import AnsibleError from ansible.playbook.play_context import PlayContext -from ansible.plugins import PluginLoader from ansible.plugins.action import ActionBase from ansible.template import Templar -from ansible.utils.unicode import to_bytes from units.mock.loader import DictDataLoader +from ansible.module_utils._text import to_bytes + python_module_replacers = b""" #!/usr/bin/python @@ -67,11 +64,13 @@ WINDOWS_ARGS = "<<INCLUDE_ANSIBLE_MODULE_JSON_ARGS>>" class DerivedActionBase(ActionBase): TRANSFERS_FILES = False + def run(self, tmp=None, task_vars=None): # We're not testing the plugin run() method, just the helper # methods ActionBase defines return super(DerivedActionBase, self).run(tmp=tmp, task_vars=task_vars) + class TestActionBase(unittest.TestCase): def test_action_base_run(self): @@ -144,7 +143,7 @@ class TestActionBase(unittest.TestCase): self.assertRaises(AnsibleError, action_base._configure_module, 'badmodule', mock_task.args) # test powershell module formatting - with patch.object(builtins, 'open', mock_open(read_data=to_bytes(powershell_module_replacers.strip(), encoding='utf-8'))) as m: + with patch.object(builtins, 'open', mock_open(read_data=to_bytes(powershell_module_replacers.strip(), encoding='utf-8'))): mock_task.action = 'win_copy' mock_task.args = dict(b=2) mock_connection.module_implementation_preferences = ('.ps1',) @@ -497,7 +496,10 @@ class TestActionBase(unittest.TestCase): action_base._connection.has_pipelining = True action_base._low_level_execute_command.return_value = dict(stdout='{"rc": 0, "stdout": "ok"}') self.assertEqual(action_base._execute_module(module_name=None, module_args=None), dict(_ansible_parsed=True, rc=0, stdout="ok", stdout_lines=['ok'])) - self.assertEqual(action_base._execute_module(module_name='foo', module_args=dict(z=9, y=8, x=7), task_vars=dict(a=1)), dict(_ansible_parsed=True, rc=0, stdout="ok", stdout_lines=['ok'])) + self.assertEqual(action_base._execute_module(module_name='foo', + module_args=dict(z=9, y=8, x=7), task_vars=dict(a=1)), + dict(_ansible_parsed=True, rc=0, stdout="ok", + stdout_lines=['ok'])) # test with needing/removing a remote tmp path action_base._configure_module.return_value = ('old', '#!/usr/bin/python', 'this is the module data', 'path') @@ -555,6 +557,7 @@ class TestActionBase(unittest.TestCase): finally: C.BECOME_ALLOW_SAME_USER = become_allow_same_user + # Note: Using nose's generator test cases here so we can't inherit from # unittest.TestCase class TestFilterNonJsonLines(object): @@ -592,4 +595,3 @@ class TestFilterNonJsonLines(object): def test_unparsable_filter_non_json_lines(self): for stdout_line in self.unparsable_cases: yield self.check_unparsable_filter_non_json_lines, stdout_line - diff --git a/test/units/plugins/connections/test_connection_ssh.py b/test/units/plugins/connections/test_connection_ssh.py index 18fba6d0f9..85c363450b 100644 --- a/test/units/plugins/connections/test_connection_ssh.py +++ b/test/units/plugins/connections/test_connection_ssh.py @@ -22,17 +22,17 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type import pipes -import sys from io import StringIO from ansible.compat.tests import unittest -from ansible.compat.tests.mock import patch, MagicMock, mock_open +from ansible.compat.tests.mock import patch, MagicMock from ansible import constants as C from ansible.errors import AnsibleError, AnsibleConnectionFailure, AnsibleFileNotFound from ansible.playbook.play_context import PlayContext from ansible.plugins.connection import ssh -from ansible.utils.unicode import to_bytes, to_unicode +from ansible.module_utils._text import to_bytes + class TestConnectionBaseClass(unittest.TestCase): @@ -277,20 +277,22 @@ class TestConnectionBaseClass(unittest.TestCase): C.ANSIBLE_SSH_RETRIES = 9 # test a regular, successful execution - conn._exec_command.return_value = (0, 'stdout', '') + conn._exec_command.return_value = (0, b'stdout', b'') res = conn.exec_command('ssh', 'some data') + self.assertEquals(res, (0, b'stdout', b''), msg='exec_command did not return what the _exec_command helper returned') # test a retry, followed by success conn._exec_command.return_value = None - conn._exec_command.side_effect = [(255, '', ''), (0, 'stdout', '')] + conn._exec_command.side_effect = [(255, '', ''), (0, b'stdout', b'')] res = conn.exec_command('ssh', 'some data') + self.assertEquals(res, (0, b'stdout', b''), msg='exec_command did not return what the _exec_command helper returned') # test multiple failures - conn._exec_command.side_effect = [(255, '', '')]*10 + conn._exec_command.side_effect = [(255, b'', b'')] * 10 self.assertRaises(AnsibleConnectionFailure, conn.exec_command, 'ssh', 'some data') # test other failure from exec_command - conn._exec_command.side_effect = [Exception('bad')]*10 + conn._exec_command.side_effect = [Exception('bad')] * 10 self.assertRaises(Exception, conn.exec_command, 'ssh', 'some data') @patch('os.path.exists') @@ -308,20 +310,22 @@ class TestConnectionBaseClass(unittest.TestCase): # test with C.DEFAULT_SCP_IF_SSH enabled C.DEFAULT_SCP_IF_SSH = True - res = conn.put_file('/path/to/in/file', '/path/to/dest/file') + conn.put_file('/path/to/in/file', '/path/to/dest/file') conn._run.assert_called_with('some command to run', None) - res = conn.put_file(u'/path/to/in/file/with/unicode-fö〩', u'/path/to/dest/file/with/unicode-fö〩') + conn.put_file(u'/path/to/in/file/with/unicode-fö〩', u'/path/to/dest/file/with/unicode-fö〩') conn._run.assert_called_with('some command to run', None) # test with C.DEFAULT_SCP_IF_SSH disabled C.DEFAULT_SCP_IF_SSH = False expected_in_data = b' '.join((b'put', to_bytes(pipes.quote('/path/to/in/file')), to_bytes(pipes.quote('/path/to/dest/file')))) + b'\n' - res = conn.put_file('/path/to/in/file', '/path/to/dest/file') + conn.put_file('/path/to/in/file', '/path/to/dest/file') conn._run.assert_called_with('some command to run', expected_in_data) - expected_in_data = b' '.join((b'put', to_bytes(pipes.quote('/path/to/in/file/with/unicode-fö〩')), to_bytes(pipes.quote('/path/to/dest/file/with/unicode-fö〩')))) + b'\n' - res = conn.put_file(u'/path/to/in/file/with/unicode-fö〩', u'/path/to/dest/file/with/unicode-fö〩') + expected_in_data = b' '.join((b'put', + to_bytes(pipes.quote('/path/to/in/file/with/unicode-fö〩')), + to_bytes(pipes.quote('/path/to/dest/file/with/unicode-fö〩')))) + b'\n' + conn.put_file(u'/path/to/in/file/with/unicode-fö〩', u'/path/to/dest/file/with/unicode-fö〩') conn._run.assert_called_with('some command to run', expected_in_data) # test that a non-zero rc raises an error @@ -346,23 +350,24 @@ class TestConnectionBaseClass(unittest.TestCase): # test with C.DEFAULT_SCP_IF_SSH enabled C.DEFAULT_SCP_IF_SSH = True - res = conn.fetch_file('/path/to/in/file', '/path/to/dest/file') + conn.fetch_file('/path/to/in/file', '/path/to/dest/file') conn._run.assert_called_with('some command to run', None) - res = conn.fetch_file(u'/path/to/in/file/with/unicode-fö〩', u'/path/to/dest/file/with/unicode-fö〩') + conn.fetch_file(u'/path/to/in/file/with/unicode-fö〩', u'/path/to/dest/file/with/unicode-fö〩') conn._run.assert_called_with('some command to run', None) # test with C.DEFAULT_SCP_IF_SSH disabled C.DEFAULT_SCP_IF_SSH = False expected_in_data = b' '.join((b'get', to_bytes(pipes.quote('/path/to/in/file')), to_bytes(pipes.quote('/path/to/dest/file')))) + b'\n' - res = conn.fetch_file('/path/to/in/file', '/path/to/dest/file') + conn.fetch_file('/path/to/in/file', '/path/to/dest/file') conn._run.assert_called_with('some command to run', expected_in_data) - expected_in_data = b' '.join((b'get', to_bytes(pipes.quote('/path/to/in/file/with/unicode-fö〩')), to_bytes(pipes.quote('/path/to/dest/file/with/unicode-fö〩')))) + b'\n' - res = conn.fetch_file(u'/path/to/in/file/with/unicode-fö〩', u'/path/to/dest/file/with/unicode-fö〩') + expected_in_data = b' '.join((b'get', + to_bytes(pipes.quote('/path/to/in/file/with/unicode-fö〩')), + to_bytes(pipes.quote('/path/to/dest/file/with/unicode-fö〩')))) + b'\n' + conn.fetch_file(u'/path/to/in/file/with/unicode-fö〩', u'/path/to/dest/file/with/unicode-fö〩') conn._run.assert_called_with('some command to run', expected_in_data) # test that a non-zero rc raises an error conn._run.return_value = (1, 'stdout', 'some errors') self.assertRaises(AnsibleError, conn.fetch_file, '/path/to/bad/file', '/remote/path/to/file') - |