From 4203850d1afe7f4bcfd32963900485f99198da6a Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Mon, 26 Oct 2015 14:23:09 -0700 Subject: Break apart a looped dependency to show a warning when parsing playbooks Display a warning when a dict key is overwritten by pyyaml Fixes #12888 --- hacking/test-module | 2 +- lib/ansible/cli/adhoc.py | 2 +- lib/ansible/cli/playbook.py | 2 +- lib/ansible/cli/vault.py | 5 +- lib/ansible/constants.py | 2 +- lib/ansible/executor/task_result.py | 2 +- lib/ansible/parsing/__init__.py | 277 +----------------------------- lib/ansible/parsing/dataloader.py | 294 ++++++++++++++++++++++++++++++++ lib/ansible/parsing/quoting.py | 30 ++++ lib/ansible/parsing/splitter.py | 10 +- lib/ansible/parsing/yaml/constructor.py | 41 ++++- lib/ansible/playbook/__init__.py | 4 +- lib/ansible/playbook/base.py | 6 +- lib/ansible/vars/__init__.py | 1 - samples/multi_queues.py | 2 +- test/units/mock/loader.py | 2 +- test/units/parsing/test_data_loader.py | 2 +- test/units/parsing/test_unquote.py | 2 +- 18 files changed, 374 insertions(+), 312 deletions(-) create mode 100644 lib/ansible/parsing/dataloader.py create mode 100644 lib/ansible/parsing/quoting.py diff --git a/hacking/test-module b/hacking/test-module index 543e803b97..412b39452c 100755 --- a/hacking/test-module +++ b/hacking/test-module @@ -36,7 +36,7 @@ import subprocess import traceback import optparse import ansible.utils.vars as utils_vars -from ansible.parsing import DataLoader +from ansible.parsing.dataloader import DataLoader from ansible.parsing.utils.jsonify import jsonify from ansible.parsing.splitter import parse_kv import ansible.executor.module_common as module_common diff --git a/lib/ansible/cli/adhoc.py b/lib/ansible/cli/adhoc.py index f2d6780c93..7d611bf1f0 100644 --- a/lib/ansible/cli/adhoc.py +++ b/lib/ansible/cli/adhoc.py @@ -27,7 +27,7 @@ from ansible.cli import CLI from ansible.errors import AnsibleError, AnsibleOptionsError from ansible.executor.task_queue_manager import TaskQueueManager from ansible.inventory import Inventory -from ansible.parsing import DataLoader +from ansible.parsing.dataloader import DataLoader from ansible.parsing.splitter import parse_kv from ansible.playbook.play import Play from ansible.plugins import get_all_plugin_loaders diff --git a/lib/ansible/cli/playbook.py b/lib/ansible/cli/playbook.py index 33414601ed..11c4f796b9 100644 --- a/lib/ansible/cli/playbook.py +++ b/lib/ansible/cli/playbook.py @@ -29,7 +29,7 @@ from ansible.cli import CLI from ansible.errors import AnsibleError, AnsibleOptionsError from ansible.executor.playbook_executor import PlaybookExecutor from ansible.inventory import Inventory -from ansible.parsing import DataLoader +from ansible.parsing.dataloader import DataLoader from ansible.utils.vars import load_extra_vars from ansible.vars import VariableManager diff --git a/lib/ansible/cli/vault.py b/lib/ansible/cli/vault.py index e4909cc255..a4329056f9 100644 --- a/lib/ansible/cli/vault.py +++ b/lib/ansible/cli/vault.py @@ -21,14 +21,11 @@ __metaclass__ = type import os import sys -import traceback -from ansible import constants as C from ansible.errors import AnsibleError, AnsibleOptionsError -from ansible.parsing import DataLoader +from ansible.parsing.dataloader import DataLoader from ansible.parsing.vault import VaultEditor from ansible.cli import CLI -from ansible.utils.display import Display class VaultCLI(CLI): """ Vault command line class """ diff --git a/lib/ansible/constants.py b/lib/ansible/constants.py index 1a919eed59..8270ef22c0 100644 --- a/lib/ansible/constants.py +++ b/lib/ansible/constants.py @@ -25,7 +25,7 @@ from string import ascii_letters, digits from ansible.compat.six import string_types from ansible.compat.six.moves import configparser -from ansible.parsing.splitter import unquote +from ansible.parsing.quoting import unquote from ansible.errors import AnsibleOptionsError # copied from utils, avoid circular reference fun :) diff --git a/lib/ansible/executor/task_result.py b/lib/ansible/executor/task_result.py index d87f9413ef..da9ab2a11a 100644 --- a/lib/ansible/executor/task_result.py +++ b/lib/ansible/executor/task_result.py @@ -19,7 +19,7 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type -from ansible.parsing import DataLoader +from ansible.parsing.dataloader import DataLoader class TaskResult: ''' diff --git a/lib/ansible/parsing/__init__.py b/lib/ansible/parsing/__init__.py index 8c1833f037..c5cbe7cf9d 100644 --- a/lib/ansible/parsing/__init__.py +++ b/lib/ansible/parsing/__init__.py @@ -1,4 +1,4 @@ -# (c) 2012-2014, Michael DeHaan +# (c) 2015, Toshio Kuratomi # # This file is part of Ansible # @@ -18,278 +18,3 @@ # Make coding more python3-ish from __future__ import (absolute_import, division, print_function) __metaclass__ = type - -import copy -import json -import os -import stat -import subprocess - -from yaml import load, YAMLError -from ansible.compat.six import text_type, string_types - -from ansible.errors import AnsibleFileNotFound, AnsibleParserError, AnsibleError -from ansible.errors.yaml_strings import YAML_SYNTAX_ERROR -from ansible.parsing.vault import VaultLib -from ansible.parsing.splitter import unquote -from ansible.parsing.yaml.loader import AnsibleLoader -from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleUnicode -from ansible.module_utils.basic import is_executable -from ansible.utils.path import unfrackpath -from ansible.utils.unicode import to_unicode - -class DataLoader(): - - ''' - The DataLoader class is used to load and parse YAML or JSON content, - either from a given file name or from a string that was previously - read in through other means. A Vault password can be specified, and - any vault-encrypted files will be decrypted. - - Data read from files will also be cached, so the file will never be - read from disk more than once. - - Usage: - - dl = DataLoader() - (or) - dl = DataLoader(vault_password='foo') - - ds = dl.load('...') - ds = dl.load_from_file('/path/to/file') - ''' - - def __init__(self): - self._basedir = '.' - self._FILE_CACHE = dict() - - # initialize the vault stuff with an empty password - self.set_vault_password(None) - - def set_vault_password(self, vault_password): - self._vault_password = vault_password - self._vault = VaultLib(password=vault_password) - - def load(self, data, file_name='', show_content=True): - ''' - Creates a python datastructure from the given data, which can be either - a JSON or YAML string. - ''' - - try: - # we first try to load this data as JSON - return json.loads(data) - except: - # if loading JSON failed for any reason, we go ahead - # and try to parse it as YAML instead - - if isinstance(data, AnsibleUnicode): - # The PyYAML's libyaml bindings use PyUnicode_CheckExact so - # they are unable to cope with our subclass. - # Unwrap and re-wrap the unicode so we can keep track of line - # numbers - new_data = text_type(data) - else: - new_data = data - try: - new_data = self._safe_load(new_data, file_name=file_name) - except YAMLError as yaml_exc: - self._handle_error(yaml_exc, file_name, show_content) - - if isinstance(data, AnsibleUnicode): - new_data = AnsibleUnicode(new_data) - new_data.ansible_pos = data.ansible_pos - return new_data - - def load_from_file(self, file_name): - ''' Loads data from a file, which can contain either JSON or YAML. ''' - - file_name = self.path_dwim(file_name) - - # if the file has already been read in and cached, we'll - # return those results to avoid more file/vault operations - if file_name in self._FILE_CACHE: - parsed_data = self._FILE_CACHE[file_name] - else: - # read the file contents and load the data structure from them - (file_data, show_content) = self._get_file_contents(file_name) - parsed_data = self.load(data=file_data, file_name=file_name, show_content=show_content) - - # cache the file contents for next time - self._FILE_CACHE[file_name] = parsed_data - - # return a deep copy here, so the cache is not affected - return copy.deepcopy(parsed_data) - - def path_exists(self, path): - path = self.path_dwim(path) - return os.path.exists(path) - - def is_file(self, path): - path = self.path_dwim(path) - return os.path.isfile(path) or path == os.devnull - - def is_directory(self, path): - path = self.path_dwim(path) - return os.path.isdir(path) - - def list_directory(self, path): - path = self.path_dwim(path) - return os.listdir(path) - - def is_executable(self, path): - '''is the given path executable?''' - path = self.path_dwim(path) - return is_executable(path) - - def _safe_load(self, stream, file_name=None): - ''' Implements yaml.safe_load(), except using our custom loader class. ''' - - loader = AnsibleLoader(stream, file_name) - try: - return loader.get_single_data() - finally: - loader.dispose() - - def _get_file_contents(self, file_name): - ''' - Reads the file contents from the given file name, and will decrypt them - if they are found to be vault-encrypted. - ''' - if not file_name or not isinstance(file_name, string_types): - raise AnsibleParserError("Invalid filename: '%s'" % str(file_name)) - - if not self.path_exists(file_name) or not self.is_file(file_name): - raise AnsibleFileNotFound("the file_name '%s' does not exist, or is not readable" % file_name) - - show_content = True - try: - with open(file_name, 'rb') as f: - data = f.read() - if self._vault.is_encrypted(data): - data = self._vault.decrypt(data) - show_content = False - - data = to_unicode(data, errors='strict') - return (data, show_content) - - except (IOError, OSError) as e: - raise AnsibleParserError("an error occurred while trying to read the file '%s': %s" % (file_name, str(e))) - - def _handle_error(self, yaml_exc, file_name, show_content): - ''' - Optionally constructs an object (AnsibleBaseYAMLObject) to encapsulate the - file name/position where a YAML exception occurred, and raises an AnsibleParserError - to display the syntax exception information. - ''' - - # if the YAML exception contains a problem mark, use it to construct - # an object the error class can use to display the faulty line - err_obj = None - if hasattr(yaml_exc, 'problem_mark'): - err_obj = AnsibleBaseYAMLObject() - err_obj.ansible_pos = (file_name, yaml_exc.problem_mark.line + 1, yaml_exc.problem_mark.column + 1) - - raise AnsibleParserError(YAML_SYNTAX_ERROR, obj=err_obj, show_content=show_content) - - def get_basedir(self): - ''' returns the current basedir ''' - return self._basedir - - def set_basedir(self, basedir): - ''' sets the base directory, used to find files when a relative path is given ''' - - if basedir is not None: - self._basedir = to_unicode(basedir) - - def path_dwim(self, given): - ''' - make relative paths work like folks expect. - ''' - - given = unquote(given) - - if given.startswith("/"): - return os.path.abspath(given) - elif given.startswith("~"): - return os.path.abspath(os.path.expanduser(given)) - else: - return os.path.abspath(os.path.join(self._basedir, given)) - - def path_dwim_relative(self, path, dirname, source): - ''' - find one file in either a role or playbook dir with or without - explicitly named dirname subdirs - - Used in action plugins and lookups to find supplemental files that - could be in either place. - ''' - - search = [] - isrole = False - - # I have full path, nothing else needs to be looked at - if source.startswith('~') or source.startswith('/'): - search.append(self.path_dwim(source)) - else: - # base role/play path + templates/files/vars + relative filename - search.append(os.path.join(path, dirname, source)) - - basedir = unfrackpath(path) - - # is it a role and if so make sure you get correct base path - if path.endswith('tasks') and os.path.exists(os.path.join(path,'main.yml')) \ - or os.path.exists(os.path.join(path,'tasks/main.yml')): - isrole = True - if path.endswith('tasks'): - basedir = unfrackpath(os.path.dirname(path)) - - cur_basedir = self._basedir - self.set_basedir(basedir) - # resolved base role/play path + templates/files/vars + relative filename - search.append(self.path_dwim(os.path.join(basedir, dirname, source))) - self.set_basedir(cur_basedir) - - if isrole and not source.endswith(dirname): - # look in role's tasks dir w/o dirname - search.append(self.path_dwim(os.path.join(basedir, 'tasks', source))) - - # try to create absolute path for loader basedir + templates/files/vars + filename - search.append(self.path_dwim(os.path.join(dirname,source))) - search.append(self.path_dwim(os.path.join(basedir, source))) - - # try to create absolute path for loader basedir + filename - search.append(self.path_dwim(source)) - - for candidate in search: - if os.path.exists(candidate): - break - - return candidate - - def read_vault_password_file(self, vault_password_file): - """ - Read a vault password from a file or if executable, execute the script and - retrieve password from STDOUT - """ - - this_path = os.path.realpath(os.path.expanduser(vault_password_file)) - if not os.path.exists(this_path): - raise AnsibleFileNotFound("The vault password file %s was not found" % this_path) - - if self.is_executable(this_path): - try: - # STDERR not captured to make it easier for users to prompt for input in their scripts - p = subprocess.Popen(this_path, stdout=subprocess.PIPE) - except OSError as e: - raise AnsibleError("Problem running vault password script %s (%s). If this is not a script, remove the executable bit from the file." % (' '.join(this_path), e)) - stdout, stderr = p.communicate() - self.set_vault_password(stdout.strip('\r\n')) - else: - try: - f = open(this_path, "rb") - self.set_vault_password(f.read().strip()) - f.close() - except (OSError, IOError) as e: - raise AnsibleError("Could not read vault password file %s: %s" % (this_path, e)) - diff --git a/lib/ansible/parsing/dataloader.py b/lib/ansible/parsing/dataloader.py new file mode 100644 index 0000000000..aaa878bb5f --- /dev/null +++ b/lib/ansible/parsing/dataloader.py @@ -0,0 +1,294 @@ +# (c) 2012-2014, Michael DeHaan +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import copy +import json +import os +import stat +import subprocess + +from yaml import load, YAMLError +from ansible.compat.six import text_type, string_types + +from ansible.errors import AnsibleFileNotFound, AnsibleParserError, AnsibleError +from ansible.errors.yaml_strings import YAML_SYNTAX_ERROR +from ansible.parsing.vault import VaultLib +from ansible.parsing.quoting import unquote +from ansible.parsing.yaml.loader import AnsibleLoader +from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleUnicode +from ansible.module_utils.basic import is_executable +from ansible.utils.path import unfrackpath +from ansible.utils.unicode import to_unicode + +class DataLoader(): + + ''' + The DataLoader class is used to load and parse YAML or JSON content, + either from a given file name or from a string that was previously + read in through other means. A Vault password can be specified, and + any vault-encrypted files will be decrypted. + + Data read from files will also be cached, so the file will never be + read from disk more than once. + + Usage: + + dl = DataLoader() + (or) + dl = DataLoader(vault_password='foo') + + ds = dl.load('...') + ds = dl.load_from_file('/path/to/file') + ''' + + def __init__(self): + self._basedir = '.' + self._FILE_CACHE = dict() + + # initialize the vault stuff with an empty password + self.set_vault_password(None) + + def set_vault_password(self, vault_password): + self._vault_password = vault_password + self._vault = VaultLib(password=vault_password) + + def load(self, data, file_name='', show_content=True): + ''' + Creates a python datastructure from the given data, which can be either + a JSON or YAML string. + ''' + + try: + # we first try to load this data as JSON + return json.loads(data) + except: + # if loading JSON failed for any reason, we go ahead + # and try to parse it as YAML instead + + if isinstance(data, AnsibleUnicode): + # The PyYAML's libyaml bindings use PyUnicode_CheckExact so + # they are unable to cope with our subclass. + # Unwrap and re-wrap the unicode so we can keep track of line + # numbers + new_data = text_type(data) + else: + new_data = data + try: + new_data = self._safe_load(new_data, file_name=file_name) + except YAMLError as yaml_exc: + self._handle_error(yaml_exc, file_name, show_content) + + if isinstance(data, AnsibleUnicode): + new_data = AnsibleUnicode(new_data) + new_data.ansible_pos = data.ansible_pos + return new_data + + def load_from_file(self, file_name): + ''' Loads data from a file, which can contain either JSON or YAML. ''' + + file_name = self.path_dwim(file_name) + + # if the file has already been read in and cached, we'll + # return those results to avoid more file/vault operations + if file_name in self._FILE_CACHE: + parsed_data = self._FILE_CACHE[file_name] + else: + # read the file contents and load the data structure from them + (file_data, show_content) = self._get_file_contents(file_name) + parsed_data = self.load(data=file_data, file_name=file_name, show_content=show_content) + + # cache the file contents for next time + self._FILE_CACHE[file_name] = parsed_data + + # return a deep copy here, so the cache is not affected + return copy.deepcopy(parsed_data) + + def path_exists(self, path): + path = self.path_dwim(path) + return os.path.exists(path) + + def is_file(self, path): + path = self.path_dwim(path) + return os.path.isfile(path) or path == os.devnull + + def is_directory(self, path): + path = self.path_dwim(path) + return os.path.isdir(path) + + def list_directory(self, path): + path = self.path_dwim(path) + return os.listdir(path) + + def is_executable(self, path): + '''is the given path executable?''' + path = self.path_dwim(path) + return is_executable(path) + + def _safe_load(self, stream, file_name=None): + ''' Implements yaml.safe_load(), except using our custom loader class. ''' + + loader = AnsibleLoader(stream, file_name) + try: + return loader.get_single_data() + finally: + loader.dispose() + + def _get_file_contents(self, file_name): + ''' + Reads the file contents from the given file name, and will decrypt them + if they are found to be vault-encrypted. + ''' + if not file_name or not isinstance(file_name, string_types): + raise AnsibleParserError("Invalid filename: '%s'" % str(file_name)) + + if not self.path_exists(file_name) or not self.is_file(file_name): + raise AnsibleFileNotFound("the file_name '%s' does not exist, or is not readable" % file_name) + + show_content = True + try: + with open(file_name, 'rb') as f: + data = f.read() + if self._vault.is_encrypted(data): + data = self._vault.decrypt(data) + show_content = False + + data = to_unicode(data, errors='strict') + return (data, show_content) + + except (IOError, OSError) as e: + raise AnsibleParserError("an error occurred while trying to read the file '%s': %s" % (file_name, str(e))) + + def _handle_error(self, yaml_exc, file_name, show_content): + ''' + Optionally constructs an object (AnsibleBaseYAMLObject) to encapsulate the + file name/position where a YAML exception occurred, and raises an AnsibleParserError + to display the syntax exception information. + ''' + + # if the YAML exception contains a problem mark, use it to construct + # an object the error class can use to display the faulty line + err_obj = None + if hasattr(yaml_exc, 'problem_mark'): + err_obj = AnsibleBaseYAMLObject() + err_obj.ansible_pos = (file_name, yaml_exc.problem_mark.line + 1, yaml_exc.problem_mark.column + 1) + + raise AnsibleParserError(YAML_SYNTAX_ERROR, obj=err_obj, show_content=show_content) + + def get_basedir(self): + ''' returns the current basedir ''' + return self._basedir + + def set_basedir(self, basedir): + ''' sets the base directory, used to find files when a relative path is given ''' + + if basedir is not None: + self._basedir = to_unicode(basedir) + + def path_dwim(self, given): + ''' + make relative paths work like folks expect. + ''' + + given = unquote(given) + + if given.startswith("/"): + return os.path.abspath(given) + elif given.startswith("~"): + return os.path.abspath(os.path.expanduser(given)) + else: + return os.path.abspath(os.path.join(self._basedir, given)) + + def path_dwim_relative(self, path, dirname, source): + ''' + find one file in either a role or playbook dir with or without + explicitly named dirname subdirs + + Used in action plugins and lookups to find supplemental files that + could be in either place. + ''' + + search = [] + isrole = False + + # I have full path, nothing else needs to be looked at + if source.startswith('~') or source.startswith('/'): + search.append(self.path_dwim(source)) + else: + # base role/play path + templates/files/vars + relative filename + search.append(os.path.join(path, dirname, source)) + + basedir = unfrackpath(path) + + # is it a role and if so make sure you get correct base path + if path.endswith('tasks') and os.path.exists(os.path.join(path,'main.yml')) \ + or os.path.exists(os.path.join(path,'tasks/main.yml')): + isrole = True + if path.endswith('tasks'): + basedir = unfrackpath(os.path.dirname(path)) + + cur_basedir = self._basedir + self.set_basedir(basedir) + # resolved base role/play path + templates/files/vars + relative filename + search.append(self.path_dwim(os.path.join(basedir, dirname, source))) + self.set_basedir(cur_basedir) + + if isrole and not source.endswith(dirname): + # look in role's tasks dir w/o dirname + search.append(self.path_dwim(os.path.join(basedir, 'tasks', source))) + + # try to create absolute path for loader basedir + templates/files/vars + filename + search.append(self.path_dwim(os.path.join(dirname,source))) + search.append(self.path_dwim(os.path.join(basedir, source))) + + # try to create absolute path for loader basedir + filename + search.append(self.path_dwim(source)) + + for candidate in search: + if os.path.exists(candidate): + break + + return candidate + + def read_vault_password_file(self, vault_password_file): + """ + Read a vault password from a file or if executable, execute the script and + retrieve password from STDOUT + """ + + this_path = os.path.realpath(os.path.expanduser(vault_password_file)) + if not os.path.exists(this_path): + raise AnsibleFileNotFound("The vault password file %s was not found" % this_path) + + if self.is_executable(this_path): + try: + # STDERR not captured to make it easier for users to prompt for input in their scripts + p = subprocess.Popen(this_path, stdout=subprocess.PIPE) + except OSError as e: + raise AnsibleError("Problem running vault password script %s (%s). If this is not a script, remove the executable bit from the file." % (' '.join(this_path), e)) + stdout, stderr = p.communicate() + self.set_vault_password(stdout.strip('\r\n')) + else: + try: + f = open(this_path, "rb") + self.set_vault_password(f.read().strip()) + f.close() + except (OSError, IOError) as e: + raise AnsibleError("Could not read vault password file %s: %s" % (this_path, e)) diff --git a/lib/ansible/parsing/quoting.py b/lib/ansible/parsing/quoting.py new file mode 100644 index 0000000000..7b94f90653 --- /dev/null +++ b/lib/ansible/parsing/quoting.py @@ -0,0 +1,30 @@ +# (c) 2014 James Cammarata, +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +def is_quoted(data): + return len(data) > 1 and data[0] == data[-1] and data[0] in ('"', "'") and data[-2] != '\\' + +def unquote(data): + ''' removes first and last quotes from a string, if the string starts and ends with the same quotes ''' + if is_quoted(data): + return data[1:-1] + return data diff --git a/lib/ansible/parsing/splitter.py b/lib/ansible/parsing/splitter.py index ef8fc89b00..3bbf6e7e9c 100644 --- a/lib/ansible/parsing/splitter.py +++ b/lib/ansible/parsing/splitter.py @@ -23,6 +23,7 @@ import re import codecs from ansible.errors import AnsibleError +from ansible.parsing.quoting import unquote # Decode escapes adapted from rspeer's answer here: # http://stackoverflow.com/questions/4020539/process-escape-sequences-in-a-string-in-python @@ -263,12 +264,3 @@ def split_args(args): raise Exception("error while splitting arguments, either an unbalanced jinja2 block or quotes") return params - -def is_quoted(data): - return len(data) > 1 and data[0] == data[-1] and data[0] in ('"', "'") and data[-2] != '\\' - -def unquote(data): - ''' removes first and last quotes from a string, if the string starts and ends with the same quotes ''' - if is_quoted(data): - return data[1:-1] - return data diff --git a/lib/ansible/parsing/yaml/constructor.py b/lib/ansible/parsing/yaml/constructor.py index d1a2a01bc2..541165a42f 100644 --- a/lib/ansible/parsing/yaml/constructor.py +++ b/lib/ansible/parsing/yaml/constructor.py @@ -19,9 +19,17 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type -from yaml.constructor import Constructor +from yaml.constructor import Constructor, ConstructorError +from yaml.nodes import MappingNode from ansible.parsing.yaml.objects import AnsibleMapping, AnsibleSequence, AnsibleUnicode +try: + from __main__ import display +except ImportError: + from ansible.utils.display import Display + display = Display() + + class AnsibleConstructor(Constructor): def __init__(self, file_name=None): self._ansible_file_name = file_name @@ -35,10 +43,33 @@ class AnsibleConstructor(Constructor): data.ansible_pos = self._node_position_info(node) def construct_mapping(self, node, deep=False): - ret = AnsibleMapping(super(Constructor, self).construct_mapping(node, deep)) - ret.ansible_pos = self._node_position_info(node) - - return ret + # This first part fixes a problem with pyyaml's handling of duplicate + # dict keys. + # from SafeConstructor + if not isinstance(node, MappingNode): + raise ConstructorError(None, None, + "expected a mapping node, but found %s" % node.id, + node.start_mark) + self.flatten_mapping(node) + mapping = AnsibleMapping() + for key_node, value_node in node.value: + key = self.construct_object(key_node, deep=deep) + try: + hash(key) + except TypeError as exc: + raise ConstructorError("while constructing a mapping", node.start_mark, + "found unacceptable key (%s)" % exc, key_node.start_mark) + + if key in mapping: + display.warning('While constructing a mapping%s found a duplicate dict key (%s). Using last value only.' % (node.start_mark, key_node.start_mark)) + + value = self.construct_object(value_node, deep=deep) + mapping[key] = value + + # Add our extra information to the returned value + mapping.ansible_pos = self._node_position_info(node) + + return mapping def construct_yaml_str(self, node): # Override the default string handling function diff --git a/lib/ansible/playbook/__init__.py b/lib/ansible/playbook/__init__.py index 37b3ee7503..888299e1d9 100644 --- a/lib/ansible/playbook/__init__.py +++ b/lib/ansible/playbook/__init__.py @@ -21,9 +21,7 @@ __metaclass__ = type import os -from ansible.errors import AnsibleError, AnsibleParserError -from ansible.parsing import DataLoader -from ansible.playbook.attribute import Attribute, FieldAttribute +from ansible.errors import AnsibleParserError from ansible.playbook.play import Play from ansible.playbook.playbook_include import PlaybookInclude from ansible.plugins import get_all_plugin_loaders diff --git a/lib/ansible/playbook/base.py b/lib/ansible/playbook/base.py index e8759c32a2..06681abe76 100644 --- a/lib/ansible/playbook/base.py +++ b/lib/ansible/playbook/base.py @@ -26,20 +26,16 @@ import uuid from functools import partial from inspect import getmembers -from io import FileIO from ansible.compat.six import iteritems, string_types, text_type from jinja2.exceptions import UndefinedError from ansible.errors import AnsibleParserError -from ansible.parsing import DataLoader +from ansible.parsing.dataloader import DataLoader from ansible.playbook.attribute import Attribute, FieldAttribute -from ansible.template import Templar from ansible.utils.boolean import boolean -from ansible.utils.debug import debug from ansible.utils.vars import combine_vars, isidentifier -from ansible.template import template class Base: diff --git a/lib/ansible/vars/__init__.py b/lib/ansible/vars/__init__.py index 7ce24f0b0a..a8972293d6 100644 --- a/lib/ansible/vars/__init__.py +++ b/lib/ansible/vars/__init__.py @@ -37,7 +37,6 @@ from ansible.cli import CLI from ansible.compat.six import string_types from ansible.errors import AnsibleError, AnsibleParserError, AnsibleUndefinedVariable, AnsibleFileNotFound from ansible.inventory.host import Host -from ansible.parsing import DataLoader from ansible.plugins import lookup_loader from ansible.plugins.cache import FactCache from ansible.template import Templar diff --git a/samples/multi_queues.py b/samples/multi_queues.py index 673bb01de6..3b647071c7 100644 --- a/samples/multi_queues.py +++ b/samples/multi_queues.py @@ -13,7 +13,7 @@ from ansible.playbook.play_context import PlayContext from ansible.playbook.task import Task from ansible.executor.task_executor import TaskExecutor from ansible.executor.task_result import TaskResult -from ansible.parsing import DataLoader +from ansible.parsing.dataloader import DataLoader from ansible.vars import VariableManager from ansible.utils.debug import debug diff --git a/test/units/mock/loader.py b/test/units/mock/loader.py index db69b862a7..2027e183c1 100644 --- a/test/units/mock/loader.py +++ b/test/units/mock/loader.py @@ -22,7 +22,7 @@ __metaclass__ = type import os from ansible.errors import AnsibleParserError -from ansible.parsing import DataLoader +from ansible.parsing.dataloader import DataLoader class DictDataLoader(DataLoader): diff --git a/test/units/parsing/test_data_loader.py b/test/units/parsing/test_data_loader.py index d4d4d9a1d5..19e3107447 100644 --- a/test/units/parsing/test_data_loader.py +++ b/test/units/parsing/test_data_loader.py @@ -26,7 +26,7 @@ from ansible.compat.tests import unittest from ansible.compat.tests.mock import patch, mock_open from ansible.errors import AnsibleParserError -from ansible.parsing import DataLoader +from ansible.parsing.dataloader import DataLoader from ansible.parsing.yaml.objects import AnsibleMapping class TestDataLoader(unittest.TestCase): diff --git a/test/units/parsing/test_unquote.py b/test/units/parsing/test_unquote.py index afb11d4e23..4e5d632066 100644 --- a/test/units/parsing/test_unquote.py +++ b/test/units/parsing/test_unquote.py @@ -23,7 +23,7 @@ __metaclass__ = type from nose import tools from ansible.compat.tests import unittest -from ansible.parsing.splitter import unquote +from ansible.parsing.quoting import unquote # Tests using nose's test generators cannot use unittest base class. -- cgit v1.2.1