Properly JSON encode AnsibleUnsafe, using a pre-processor (#60602)

* Properly JSON encode AnsibleUnsafe, using a pre-processor. Fixes #47295

* Add AnsibleUnsafe json tests

* Require preprocess_unsafe to be enabled for that functionality

* Support older json

* sort keys in tests

* Decouple AnsibleJSONEncoder from isinstance checks in preparation to move to module_utils

* Move AnsibleJSONEncoder to module_utils, consolidate instances

* add missing boilerplate

* remove removed.py from ignore

1 files changed, 70 insertions, 0 deletions

diff --git a/lib/ansible/module_utils/common/json.py b/lib/ansible/module_utils/common/json.py
new file mode 100644
index 0000000000..7da31f4598
--- /dev/null
+++ b/lib/ansible/module_utils/common/json.py
@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2019 Ansible Project
+# Simplified BSD License (see licenses/simplified_bsd.txt or https://opensource.org/licenses/BSD-2-Clause)
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import json
+
+import datetime
+
+from ansible.module_utils._text import to_text
+from ansible.module_utils.common._collections_compat import Mapping
+from ansible.module_utils.common.collections import is_sequence
+
+
+def _preprocess_unsafe_encode(value):
+    """Recursively preprocess a data structure converting instances of ``AnsibleUnsafe``
+    into their JSON dict representations
+
+    Used in ``AnsibleJSONEncoder.iterencode``
+    """
+    if getattr(value, '__UNSAFE__', False) and not getattr(value, '__ENCRYPTED__', False):
+        value = {'__ansible_unsafe': to_text(value, errors='surrogate_or_strict', nonstring='strict')}
+    elif is_sequence(value):
+        value = [_preprocess_unsafe_encode(v) for v in value]
+    elif isinstance(value, Mapping):
+        value = dict((k, _preprocess_unsafe_encode(v)) for k, v in value.items())
+
+    return value
+
+
+class AnsibleJSONEncoder(json.JSONEncoder):
+    '''
+    Simple encoder class to deal with JSON encoding of Ansible internal types
+    '''
+
+    def __init__(self, preprocess_unsafe=False, **kwargs):
+        self._preprocess_unsafe = preprocess_unsafe
+        super(AnsibleJSONEncoder, self).__init__(**kwargs)
+
+    # NOTE: ALWAYS inform AWS/Tower when new items get added as they consume them downstream via a callback
+    def default(self, o):
+        if getattr(o, '__ENCRYPTED__', False):
+            # vault object
+            value = {'__ansible_vault': to_text(o._ciphertext, errors='surrogate_or_strict', nonstring='strict')}
+        elif getattr(o, '__UNSAFE__', False):
+            # unsafe object, this will never be triggered, see ``AnsibleJSONEncoder.iterencode``
+            value = {'__ansible_unsafe': to_text(o, errors='surrogate_or_strict', nonstring='strict')}
+        elif isinstance(o, Mapping):
+            # hostvars and other objects
+            value = dict(o)
+        elif isinstance(o, (datetime.date, datetime.datetime)):
+            # date object
+            value = o.isoformat()
+        else:
+            # use default encoder
+            value = super(AnsibleJSONEncoder, self).default(o)
+        return value
+
+    def iterencode(self, o, **kwargs):
+        """Custom iterencode, primarily design to handle encoding ``AnsibleUnsafe``
+        as the ``AnsibleUnsafe`` subclasses inherit from string types and
+        ``json.JSONEncoder`` does not support custom encoders for string types
+        """
+        if self._preprocess_unsafe:
+            o = _preprocess_unsafe_encode(o)
+
+        return super(AnsibleJSONEncoder, self).iterencode(o, **kwargs)