Make AnsibleVaultEncryptedUnicode work more like a string (#67823)

* Make AnsibleVaultEncryptedUnicode work more like a string. Fixes #24425

* Remove debugging

* Wrap some things

* Reduce diff

* data should always result in text

* add tests

* Don't just copy and paste, kids

* Add eq and ne back

* Go full UserString copy/paste

* Various version related fixes

* Remove trailing newline

* py2v3

* Add a test that can evaluate whether a variable is vault encrypted

* map was introduces in jinja2 2.7

* moar jinja

* type fix

Co-Authored-By: Sam Doran <sdoran@redhat.com>

* Remove duplicate __hash__

* Fix typo

* Add changelog fragment

* ci_complete

Co-authored-by: Sam Doran <sdoran@redhat.com>

4 files changed, 265 insertions, 12 deletions

diff --git a/lib/ansible/parsing/vault/__init__.py b/lib/ansible/parsing/vault/__init__.py
index 81939af668..6cf5dc72b7 100644
--- a/lib/ansible/parsing/vault/__init__.py
+++ b/lib/ansible/parsing/vault/__init__.py
@@ -599,6 +599,10 @@ class VaultLib:
         self.cipher_name = None
         self.b_version = b'1.2'
 
+    @staticmethod
+    def is_encrypted(vaulttext):
+        return is_encrypted(vaulttext)
+
     def encrypt(self, plaintext, secret=None, vault_id=None):
         """Vault encrypt a piece of data.
 
diff --git a/lib/ansible/parsing/yaml/objects.py b/lib/ansible/parsing/yaml/objects.py
index ef11e6740b..9c93006d91 100644
--- a/lib/ansible/parsing/yaml/objects.py
+++ b/lib/ansible/parsing/yaml/objects.py
@@ -19,9 +19,13 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+import string
+import sys as _sys
+
 import sys
 import yaml
 
+from ansible.module_utils.common._collections_compat import Sequence
 from ansible.module_utils.six import text_type
 from ansible.module_utils._text import to_bytes, to_text, to_native
 
@@ -79,9 +83,8 @@ class AnsibleSequence(AnsibleBaseYAMLObject, list):
     pass
 
 
-# Unicode like object that is not evaluated (decrypted) until it needs to be
-# TODO: is there a reason these objects are subclasses for YAMLObject?
-class AnsibleVaultEncryptedUnicode(yaml.YAMLObject, AnsibleBaseYAMLObject):
+class AnsibleVaultEncryptedUnicode(Sequence, AnsibleBaseYAMLObject):
+    '''Unicode like object that is not evaluated (decrypted) until it needs to be'''
     __UNSAFE__ = True
     __ENCRYPTED__ = True
     yaml_tag = u'!vault'
@@ -113,30 +116,31 @@ class AnsibleVaultEncryptedUnicode(yaml.YAMLObject, AnsibleBaseYAMLObject):
     @property
     def data(self):
         if not self.vault:
-            return self._ciphertext
+            return to_text(self._ciphertext)
         return to_text(self.vault.decrypt(self._ciphertext))
 
     @data.setter
     def data(self, value):
-        self._ciphertext = value
+        self._ciphertext = to_bytes(value)
 
-    def __repr__(self):
-        return repr(self.data)
+    def is_encrypted(self):
+        return self.vault and self.vault.is_encrypted(self._ciphertext)
 
-    # Compare a regular str/text_type with the decrypted hypertext
     def __eq__(self, other):
         if self.vault:
             return other == self.data
         return False
 
-    def __hash__(self):
-        return id(self)
-
     def __ne__(self, other):
         if self.vault:
             return other != self.data
         return True
 
+    def __reversed__(self):
+        # This gets inerhited from ``collections.Sequence`` which returns a generator
+        # make this act more like the string implementation
+        return to_text(self[::-1], errors='surrogate_or_strict')
+
     def __str__(self):
         return to_native(self.data, errors='surrogate_or_strict')
 
@@ -144,4 +148,232 @@ class AnsibleVaultEncryptedUnicode(yaml.YAMLObject, AnsibleBaseYAMLObject):
         return to_text(self.data, errors='surrogate_or_strict')
 
     def encode(self, encoding=None, errors=None):
-        return self.data.encode(encoding, errors)
+        return to_bytes(self.data, encoding=encoding, errors=errors)
+
+    # Methods below are a copy from ``collections.UserString``
+    # Some are copied as is, where others are modified to not
+    # auto wrap with ``self.__class__``
+    def __repr__(self):
+        return repr(self.data)
+
+    def __int__(self, base=10):
+        return int(self.data, base=base)
+
+    def __float__(self):
+        return float(self.data)
+
+    def __complex__(self):
+        return complex(self.data)
+
+    def __hash__(self):
+        return hash(self.data)
+
+    # This breaks vault, do not define it, we cannot satisfy this
+    # def __getnewargs__(self):
+    #    return (self.data[:],)
+
+    def __lt__(self, string):
+        if isinstance(string, AnsibleVaultEncryptedUnicode):
+            return self.data < string.data
+        return self.data < string
+
+    def __le__(self, string):
+        if isinstance(string, AnsibleVaultEncryptedUnicode):
+            return self.data <= string.data
+        return self.data <= string
+
+    def __gt__(self, string):
+        if isinstance(string, AnsibleVaultEncryptedUnicode):
+            return self.data > string.data
+        return self.data > string
+
+    def __ge__(self, string):
+        if isinstance(string, AnsibleVaultEncryptedUnicode):
+            return self.data >= string.data
+        return self.data >= string
+
+    def __contains__(self, char):
+        if isinstance(char, AnsibleVaultEncryptedUnicode):
+            char = char.data
+        return char in self.data
+
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, index):
+        return self.data[index]
+
+    def __getslice__(self, start, end):
+        start = max(start, 0)
+        end = max(end, 0)
+        return self.data[start:end]
+
+    def __add__(self, other):
+        if isinstance(other, AnsibleVaultEncryptedUnicode):
+            return self.data + other.data
+        elif isinstance(other, text_type):
+            return self.data + other
+        return self.data + to_text(other)
+
+    def __radd__(self, other):
+        if isinstance(other, text_type):
+            return other + self.data
+        return to_text(other) + self.data
+
+    def __mul__(self, n):
+        return self.data * n
+
+    __rmul__ = __mul__
+
+    def __mod__(self, args):
+        return self.data % args
+
+    def __rmod__(self, template):
+        return to_text(template) % self
+
+    # the following methods are defined in alphabetical order:
+    def capitalize(self):
+        return self.data.capitalize()
+
+    def casefold(self):
+        return self.data.casefold()
+
+    def center(self, width, *args):
+        return self.data.center(width, *args)
+
+    def count(self, sub, start=0, end=_sys.maxsize):
+        if isinstance(sub, AnsibleVaultEncryptedUnicode):
+            sub = sub.data
+        return self.data.count(sub, start, end)
+
+    def endswith(self, suffix, start=0, end=_sys.maxsize):
+        return self.data.endswith(suffix, start, end)
+
+    def expandtabs(self, tabsize=8):
+        return self.data.expandtabs(tabsize)
+
+    def find(self, sub, start=0, end=_sys.maxsize):
+        if isinstance(sub, AnsibleVaultEncryptedUnicode):
+            sub = sub.data
+        return self.data.find(sub, start, end)
+
+    def format(self, *args, **kwds):
+        return self.data.format(*args, **kwds)
+
+    def format_map(self, mapping):
+        return self.data.format_map(mapping)
+
+    def index(self, sub, start=0, end=_sys.maxsize):
+        return self.data.index(sub, start, end)
+
+    def isalpha(self):
+        return self.data.isalpha()
+
+    def isalnum(self):
+        return self.data.isalnum()
+
+    def isascii(self):
+        return self.data.isascii()
+
+    def isdecimal(self):
+        return self.data.isdecimal()
+
+    def isdigit(self):
+        return self.data.isdigit()
+
+    def isidentifier(self):
+        return self.data.isidentifier()
+
+    def islower(self):
+        return self.data.islower()
+
+    def isnumeric(self):
+        return self.data.isnumeric()
+
+    def isprintable(self):
+        return self.data.isprintable()
+
+    def isspace(self):
+        return self.data.isspace()
+
+    def istitle(self):
+        return self.data.istitle()
+
+    def isupper(self):
+        return self.data.isupper()
+
+    def join(self, seq):
+        return self.data.join(seq)
+
+    def ljust(self, width, *args):
+        return self.data.ljust(width, *args)
+
+    def lower(self):
+        return self.data.lower()
+
+    def lstrip(self, chars=None):
+        return self.data.lstrip(chars)
+
+    try:
+        # PY3
+        maketrans = str.maketrans
+    except AttributeError:
+        # PY2
+        maketrans = string.maketrans
+
+    def partition(self, sep):
+        return self.data.partition(sep)
+
+    def replace(self, old, new, maxsplit=-1):
+        if isinstance(old, AnsibleVaultEncryptedUnicode):
+            old = old.data
+        if isinstance(new, AnsibleVaultEncryptedUnicode):
+            new = new.data
+        return self.data.replace(old, new, maxsplit)
+
+    def rfind(self, sub, start=0, end=_sys.maxsize):
+        if isinstance(sub, AnsibleVaultEncryptedUnicode):
+            sub = sub.data
+        return self.data.rfind(sub, start, end)
+
+    def rindex(self, sub, start=0, end=_sys.maxsize):
+        return self.data.rindex(sub, start, end)
+
+    def rjust(self, width, *args):
+        return self.data.rjust(width, *args)
+
+    def rpartition(self, sep):
+        return self.data.rpartition(sep)
+
+    def rstrip(self, chars=None):
+        return self.data.rstrip(chars)
+
+    def split(self, sep=None, maxsplit=-1):
+        return self.data.split(sep, maxsplit)
+
+    def rsplit(self, sep=None, maxsplit=-1):
+        return self.data.rsplit(sep, maxsplit)
+
+    def splitlines(self, keepends=False):
+        return self.data.splitlines(keepends)
+
+    def startswith(self, prefix, start=0, end=_sys.maxsize):
+        return self.data.startswith(prefix, start, end)
+
+    def strip(self, chars=None):
+        return self.data.strip(chars)
+
+    def swapcase(self):
+        return self.data.swapcase()
+
+    def title(self):
+        return self.data.title()
+
+    def translate(self, *args):
+        return self.data.translate(*args)
+
+    def upper(self):
+        return self.data.upper()
+
+    def zfill(self, width):
+        return self.data.zfill(width)
diff --git a/lib/ansible/plugins/filter/core.py b/lib/ansible/plugins/filter/core.py
index 48fab01174..9eecd79b79 100644
--- a/lib/ansible/plugins/filter/core.py
+++ b/lib/ansible/plugins/filter/core.py
@@ -134,6 +134,9 @@ def regex_replace(value='', pattern='', replacement='', ignorecase=False, multil
 
 def regex_findall(value, regex, multiline=False, ignorecase=False):
     ''' Perform re.findall and return the list of matches '''
+
+    value = to_text(value, errors='surrogate_or_strict', nonstring='simplerepr')
+
     flags = 0
     if ignorecase:
         flags |= re.I
@@ -145,6 +148,8 @@ def regex_findall(value, regex, multiline=False, ignorecase=False):
 def regex_search(value, regex, *args, **kwargs):
     ''' Perform re.search and return the list of matches or a backref '''
 
+    value = to_text(value, errors='surrogate_or_strict', nonstring='simplerepr')
+
     groups = list()
     for arg in args:
         if arg.startswith('\\g'):
@@ -184,6 +189,7 @@ def ternary(value, true_val, false_val, none_val=None):
 
 
 def regex_escape(string, re_type='python'):
+    string = to_text(string, errors='surrogate_or_strict', nonstring='simplerepr')
     '''Escape all regular expressions special characters from STRING.'''
     if re_type == 'python':
         return re.escape(string)
diff --git a/lib/ansible/plugins/test/core.py b/lib/ansible/plugins/test/core.py
index f88a5d931b..40733a1402 100644
--- a/lib/ansible/plugins/test/core.py
+++ b/lib/ansible/plugins/test/core.py
@@ -128,6 +128,14 @@ def regex(value='', pattern='', ignorecase=False, multiline=False, match_type='s
     return bool(getattr(_re, match_type, 'search')(value))
 
 
+def vault_encrypted(value):
+    """Evaulate whether a variable is a single vault encrypted value
+
+    .. versionadded:: 2.10
+    """
+    return getattr(value, '__ENCRYPTED__', False) and value.is_encrypted()
+
+
 def match(value, pattern='', ignorecase=False, multiline=False):
     ''' Perform a `re.match` returning a boolean '''
     return regex(value, pattern, ignorecase, multiline, 'match')
@@ -236,4 +244,7 @@ class TestModule(object):
             # truthiness
             'truthy': truthy,
             'falsy': falsy,
+
+            # vault
+            'vault_encrypted': vault_encrypted,
         }