py3: Monkey-patch json.loads to accept bytes on py35

I'm tired of creating code churn where I just slap

    .decode("nearly arbitrary choice of encoding")

in a bunch of places.

Change-Id: I79b2bc59fed130ca537e96c1074212861d7db6b8

8 files changed, 44 insertions, 9 deletions

diff --git a/swift/__init__.py b/swift/__init__.py
index 9d0e8896f..f9f093132 100644
--- a/swift/__init__.py
+++ b/swift/__init__.py
@@ -14,6 +14,7 @@
 # limitations under the License.
 
 import os
+import sys
 import gettext
 
 import pkg_resources
@@ -39,3 +40,37 @@ _t = gettext.translation('swift', localedir=_localedir, fallback=True)
 
 def gettext_(msg):
     return _t.gettext(msg)
+
+
+if (3, 0) <= sys.version_info[:2] <= (3, 5):
+    # In the development of py3, json.loads() stopped accepting byte strings
+    # for a while. https://bugs.python.org/issue17909 got fixed for py36, but
+    # since it was termed an enhancement and not a regression, we don't expect
+    # any backports. At the same time, it'd be better if we could avoid
+    # leaving a whole bunch of json.loads(resp.body.decode(...)) scars in the
+    # code that'd probably persist even *after* we drop support for 3.5 and
+    # earlier. So, monkey patch stdlib.
+    import json
+    if not getattr(json.loads, 'patched_to_decode', False):
+        class JsonLoadsPatcher(object):
+            def __init__(self, orig):
+                self._orig = orig
+
+            def __call__(self, s, **kw):
+                if isinstance(s, bytes):
+                    # No fancy byte-order mark detection for us; just assume
+                    # UTF-8 and raise a UnicodeDecodeError if appropriate.
+                    s = s.decode('utf8')
+                return self._orig(s, **kw)
+
+            def __getattribute__(self, attr):
+                if attr == 'patched_to_decode':
+                    return True
+                if attr == '_orig':
+                    return super().__getattribute__(attr)
+                # Pass through all other attrs to the original; among other
+                # things, this preserves doc strings, etc.
+                return getattr(self._orig, attr)
+
+        json.loads = JsonLoadsPatcher(json.loads)
+        del JsonLoadsPatcher
diff --git a/swift/common/direct_client.py b/swift/common/direct_client.py
index 2d4e2ce6a..174b61601 100644
--- a/swift/common/direct_client.py
+++ b/swift/common/direct_client.py
@@ -174,7 +174,7 @@ def _get_direct_account_container(path, stype, node, part,
     if resp.status == HTTP_NO_CONTENT:
         resp.read()
         return resp_headers, []
-    return resp_headers, json.loads(resp.read().decode('ascii'))
+    return resp_headers, json.loads(resp.read())
 
 
 def gen_headers(hdrs_in=None, add_ts=True):
diff --git a/swift/common/internal_client.py b/swift/common/internal_client.py
index d4cc2d19e..95b52ae02 100644
--- a/swift/common/internal_client.py
+++ b/swift/common/internal_client.py
@@ -298,7 +298,7 @@ class InternalClient(object):
                 if resp.status_int >= HTTP_MULTIPLE_CHOICES:
                     b''.join(resp.app_iter)
                 break
-            data = json.loads(resp.body.decode('ascii'))
+            data = json.loads(resp.body)
             if not data:
                 break
             for item in data:
@@ -844,7 +844,7 @@ class SimpleClient(object):
         body = conn.read()
         info = conn.info()
         try:
-            body_data = json.loads(body.decode('ascii'))
+            body_data = json.loads(body)
         except ValueError:
             body_data = None
         trans_stop = time()
diff --git a/swift/common/memcached.py b/swift/common/memcached.py
index 4b1a879a4..8583cafb4 100644
--- a/swift/common/memcached.py
+++ b/swift/common/memcached.py
@@ -315,7 +315,7 @@ class MemcacheRing(object):
                                 else:
                                     value = None
                             elif int(line[2]) & JSON_FLAG:
-                                value = json.loads(value.decode('ascii'))
+                                value = json.loads(value)
                             fp.readline()
                         line = fp.readline().strip().split()
                     self._return_conn(server, fp, sock)
@@ -484,7 +484,7 @@ class MemcacheRing(object):
                                 else:
                                     value = None
                             elif int(line[2]) & JSON_FLAG:
-                                value = json.loads(value.decode('ascii'))
+                                value = json.loads(value)
                             responses[line[1]] = value
                             fp.readline()
                         line = fp.readline().strip().split()
diff --git a/swift/common/middleware/listing_formats.py b/swift/common/middleware/listing_formats.py
index d70ac484e..7fcd7ff33 100644
--- a/swift/common/middleware/listing_formats.py
+++ b/swift/common/middleware/listing_formats.py
@@ -185,7 +185,7 @@ class ListingFilter(object):
 
         body = b''.join(resp_iter)
         try:
-            listing = json.loads(body.decode('ascii'))
+            listing = json.loads(body)
             # Do a couple sanity checks
             if not isinstance(listing, list):
                 raise ValueError
diff --git a/swift/common/middleware/symlink.py b/swift/common/middleware/symlink.py
index a63eaac77..94e6c8edf 100644
--- a/swift/common/middleware/symlink.py
+++ b/swift/common/middleware/symlink.py
@@ -295,7 +295,7 @@ class SymlinkContainerContext(WSGIContext):
         """
         with closing_if_possible(resp_iter):
             resp_body = b''.join(resp_iter)
-        body_json = json.loads(resp_body.decode('ascii'))
+        body_json = json.loads(resp_body)
         swift_version, account, _junk = split_path(req.path, 2, 3, True)
         new_body = json.dumps(
             [self._extract_symlink_path_json(obj_dict, swift_version, account)
diff --git a/swift/common/ring/ring.py b/swift/common/ring/ring.py
index 3f1a484e4..a28d97d2f 100644
--- a/swift/common/ring/ring.py
+++ b/swift/common/ring/ring.py
@@ -78,7 +78,7 @@ class RingData(object):
         """
 
         json_len, = struct.unpack('!I', gz_file.read(4))
-        ring_dict = json.loads(gz_file.read(json_len).decode('ascii'))
+        ring_dict = json.loads(gz_file.read(json_len))
         ring_dict['replica2part2dev_id'] = []
 
         if metadata_only:
diff --git a/swift/common/utils.py b/swift/common/utils.py
index 6289511e1..9e45ceb36 100644
--- a/swift/common/utils.py
+++ b/swift/common/utils.py
@@ -3477,7 +3477,7 @@ def dump_recon_cache(cache_dict, cache_file, logger, lock_timeout=2,
             try:
                 existing_entry = cf.readline()
                 if existing_entry:
-                    cache_entry = json.loads(existing_entry.decode('utf8'))
+                    cache_entry = json.loads(existing_entry)
             except ValueError:
                 # file doesn't have a valid entry, we'll recreate it
                 pass