Merge "Properly read py2 object metadata on py3"

4 files changed, 256 insertions, 33 deletions

diff --git a/swift/cli/info.py b/swift/cli/info.py
index 7826a17b8..d99fb3b19 100644
--- a/swift/cli/info.py
+++ b/swift/cli/info.py
@@ -30,6 +30,7 @@ from swift.container.backend import ContainerBroker, DATADIR as CBDATADIR
 from swift.obj.diskfile import get_data_dir, read_metadata, DATADIR_BASE, \
     extract_policy
 from swift.common.storage_policy import POLICIES
+from swift.common.swob import wsgi_to_str
 from swift.common.middleware.crypto.crypto_utils import load_crypto_meta
 from swift.common.utils import md5
 
@@ -537,6 +538,8 @@ def print_obj(datafile, check_etag=True, swift_dir='/etc/swift',
         except EOFError:
             print("Invalid metadata")
             raise InfoSystemExit()
+        metadata = {wsgi_to_str(k): v if k == 'name' else wsgi_to_str(v)
+                    for k, v in metadata.items()}
 
         etag = metadata.pop('ETag', '')
         length = metadata.pop('Content-Length', '')
diff --git a/swift/obj/diskfile.py b/swift/obj/diskfile.py
index efd897907..8b6b7d01b 100644
--- a/swift/obj/diskfile.py
+++ b/swift/obj/diskfile.py
@@ -167,24 +167,36 @@ def _encode_metadata(metadata):
     return dict(((encode_str(k), encode_str(v)) for k, v in metadata.items()))
 
 
-def _decode_metadata(metadata):
+def _decode_metadata(metadata, metadata_written_by_py3):
     """
     Given a metadata dict from disk, convert keys and values to native strings.
 
     :param metadata: a dict
+    :param metadata_written_by_py3:
     """
     if six.PY2:
-        def to_str(item):
+        def to_str(item, is_name=False):
+            # For years, py2 and py3 handled non-ascii metadata differently;
+            # see https://bugs.launchpad.net/swift/+bug/2012531
+            if metadata_written_by_py3 and not is_name:
+                # do our best to read new-style data replicated from a py3 node
+                item = item.decode('utf8').encode('latin1')
             if isinstance(item, six.text_type):
                 return item.encode('utf8')
             return item
     else:
-        def to_str(item):
+        def to_str(item, is_name=False):
+            # For years, py2 and py3 handled non-ascii metadata differently;
+            # see https://bugs.launchpad.net/swift/+bug/2012531
+            if not metadata_written_by_py3 and isinstance(item, bytes) \
+                    and not is_name:
+                # do our best to read old py2 data
+                item = item.decode('latin1')
             if isinstance(item, six.binary_type):
                 return item.decode('utf8', 'surrogateescape')
             return item
 
-    return dict(((to_str(k), to_str(v)) for k, v in metadata.items()))
+    return {to_str(k): to_str(v, k == b'name') for k, v in metadata.items()}
 
 
 def read_metadata(fd, add_missing_checksum=False):
@@ -238,6 +250,7 @@ def read_metadata(fd, add_missing_checksum=False):
                 "stored checksum='%s', computed='%s'" % (
                     fd, metadata_checksum, computed_checksum))
 
+    metadata_written_by_py3 = (b'_codecs\nencode' in metadata[:32])
     # strings are utf-8 encoded when written, but have not always been
     # (see https://bugs.launchpad.net/swift/+bug/1678018) so encode them again
     # when read
@@ -245,7 +258,7 @@ def read_metadata(fd, add_missing_checksum=False):
         metadata = pickle.loads(metadata)
     else:
         metadata = pickle.loads(metadata, encoding='bytes')
-    return _decode_metadata(metadata)
+    return _decode_metadata(metadata, metadata_written_by_py3)
 
 
 def write_metadata(fd, metadata, xattr_size=65536):
diff --git a/test/unit/cli/test_info.py b/test/unit/cli/test_info.py
index 80813db87..85fb79819 100644
--- a/test/unit/cli/test_info.py
+++ b/test/unit/cli/test_info.py
@@ -1219,7 +1219,7 @@ class TestPrintObjFullMeta(TestCliInfoBase):
             md = {'name': '/AUTH_admin/c/obj',
                   'Content-Type': 'application/octet-stream',
                   'ETag': 'd41d8cd98f00b204e9800998ecf8427e',
-                  'Content-Length': 0}
+                  'Content-Length': '0'}
             write_metadata(fp, md)
 
     def test_print_obj(self):
@@ -1256,7 +1256,7 @@ class TestPrintObjFullMeta(TestCliInfoBase):
             md = {'name': '/AUTH_admin/c/obj',
                   'Content-Type': 'application/octet-stream',
                   'ETag': 'd41d8cd98f00b204e9800998ecf8427e',
-                  'Content-Length': 0}
+                  'Content-Length': '0'}
             write_metadata(fp, md)
 
         object_ring = ring.Ring(self.testdir, ring_name='object-2')
@@ -1297,7 +1297,7 @@ class TestPrintObjFullMeta(TestCliInfoBase):
             md = {'name': '/AUTH_admin/c/obj',
                   'Content-Type': 'application/octet-stream',
                   'ETag': 'd41d8cd98f00b204e9800998ecf8427e',
-                  'Content-Length': 0}
+                  'Content-Length': '0'}
             write_metadata(fp, md)
 
         object_ring = ring.Ring(self.testdir, ring_name='object-3')
@@ -1368,7 +1368,7 @@ class TestPrintObjFullMeta(TestCliInfoBase):
             md = {'name': '/AUTH_admin/c/obj',
                   'Content-Type': 'application/octet-stream',
                   'ETag': 'badetag',
-                  'Content-Length': 0}
+                  'Content-Length': '0'}
             write_metadata(fp, md)
 
         out = StringIO()
@@ -1703,5 +1703,5 @@ class TestPrintObjWeirdPath(TestPrintObjFullMeta):
             md = {'name': '/AUTH_admin/c/obj',
                   'Content-Type': 'application/octet-stream',
                   'ETag': 'd41d8cd98f00b204e9800998ecf8427e',
-                  'Content-Length': 0}
+                  'Content-Length': '0'}
             write_metadata(fp, md)
diff --git a/test/unit/obj/test_diskfile.py b/test/unit/obj/test_diskfile.py
index 327d860e5..8ee4edb45 100644
--- a/test/unit/obj/test_diskfile.py
+++ b/test/unit/obj/test_diskfile.py
@@ -15,6 +15,8 @@
 
 """Tests for swift.obj.diskfile"""
 
+from __future__ import print_function
+import binascii
 import six.moves.cPickle as pickle
 import os
 import errno
@@ -28,6 +30,7 @@ import uuid
 import xattr
 import re
 import six
+import sys
 from collections import defaultdict
 from random import shuffle, randint
 from shutil import rmtree
@@ -554,6 +557,233 @@ class TestDiskFileModuleMethods(unittest.TestCase):
         exp_dir = '/srv/node/sdb5/objects-1/123'
         self.assertEqual(part_dir, exp_dir)
 
+    def test_can_read_old_meta(self):
+        # outputs taken from `xattr -l <diskfile>`
+        cases = {
+            'python_2.7.18_swift_2.13_replicated': '''
+0000   80 02 7D 71 01 28 55 0E 43 6F 6E 74 65 6E 74 2D    ..}q.(U.Content-
+0010   4C 65 6E 67 74 68 71 02 55 02 31 33 55 04 6E 61    Lengthq.U.13U.na
+0020   6D 65 71 03 55 12 2F 41 55 54 48 5F 74 65 73 74    meq.U./AUTH_test
+0030   2F E2 98 83 2F E2 98 83 71 04 55 13 58 2D 4F 62    /.../...q.U.X-Ob
+0040   6A 65 63 74 2D 4D 65 74 61 2D 4D 74 69 6D 65 55    ject-Meta-MtimeU
+0050   11 31 36 38 32 39 35 39 38 37 34 2E 37 35 36 32    .1682959874.7562
+0060   30 35 71 05 55 04 45 54 61 67 71 06 55 20 36 62    05q.U.ETagq.U 6b
+0070   37 64 39 61 31 63 35 64 31 36 37 63 63 35 30 30    7d9a1c5d167cc500
+0080   33 37 66 32 39 66 32 39 30 62 62 33 37 35 71 07    37f29f290bb375q.
+0090   55 0B 58 2D 54 69 6D 65 73 74 61 6D 70 71 08 55    U.X-Timestampq.U
+00A0   10 31 36 38 32 39 36 32 36 35 31 2E 39 37 34 39    .1682962651.9749
+00B0   34 55 11 58 2D 4F 62 6A 65 63 74 2D 4D 65 74 61    4U.X-Object-Meta
+00C0   2D E2 98 83 55 03 E2 98 83 71 09 55 0C 43 6F 6E    -...U....q.U.Con
+00D0   74 65 6E 74 2D 54 79 70 65 71 0A 55 18 61 70 70    tent-Typeq.U.app
+00E0   6C 69 63 61 74 69 6F 6E 2F 6F 63 74 65 74 2D 73    lication/octet-s
+00F0   74 72 65 61 6D 71 0B 75 2E                         treamq.u.
+            ''',
+            'python_2.7.18_swift_2.13_ec': '''
+0000   80 02 7D 71 01 28 55 0E 43 6F 6E 74 65 6E 74 2D    ..}q.(U.Content-
+0010   4C 65 6E 67 74 68 71 02 55 02 38 34 55 04 6E 61    Lengthq.U.84U.na
+0020   6D 65 71 03 55 12 2F 41 55 54 48 5F 74 65 73 74    meq.U./AUTH_test
+0030   2F E2 98 83 2F E2 98 83 71 04 58 1E 00 00 00 58    /.../...q.X....X
+0040   2D 4F 62 6A 65 63 74 2D 53 79 73 6D 65 74 61 2D    -Object-Sysmeta-
+0050   45 63 2D 46 72 61 67 2D 49 6E 64 65 78 71 05 55    Ec-Frag-Indexq.U
+0060   01 35 55 13 58 2D 4F 62 6A 65 63 74 2D 4D 65 74    .5U.X-Object-Met
+0070   61 2D 4D 74 69 6D 65 55 11 31 36 38 32 39 35 39    a-MtimeU.1682959
+0080   38 37 34 2E 37 35 36 32 30 35 71 06 58 22 00 00    874.756205q.X"..
+0090   00 58 2D 4F 62 6A 65 63 74 2D 53 79 73 6D 65 74    .X-Object-Sysmet
+00A0   61 2D 45 63 2D 43 6F 6E 74 65 6E 74 2D 4C 65 6E    a-Ec-Content-Len
+00B0   67 74 68 71 07 55 02 31 33 71 08 58 18 00 00 00    gthq.U.13q.X....
+00C0   58 2D 4F 62 6A 65 63 74 2D 53 79 73 6D 65 74 61    X-Object-Sysmeta
+00D0   2D 45 63 2D 45 74 61 67 71 09 55 20 36 62 37 64    -Ec-Etagq.U 6b7d
+00E0   39 61 31 63 35 64 31 36 37 63 63 35 30 30 33 37    9a1c5d167cc50037
+00F0   66 32 39 66 32 39 30 62 62 33 37 35 71 0A 55 04    f29f290bb375q.U.
+0100   45 54 61 67 71 0B 55 20 65 32 66 64 34 33 30 65    ETagq.U e2fd430e
+0110   61 66 37 32 32 33 63 32 35 30 33 63 34 65 38 33    af7223c2503c4e83
+0120   30 31 63 66 66 33 37 63 71 0C 55 0B 58 2D 54 69    01cff37cq.U.X-Ti
+0130   6D 65 73 74 61 6D 70 71 0D 55 10 31 36 38 32 39    mestampq.U.16829
+0140   36 32 32 36 32 2E 31 36 31 39 39 55 11 58 2D 4F    62262.16199U.X-O
+0150   62 6A 65 63 74 2D 4D 65 74 61 2D E2 98 83 55 03    bject-Meta-...U.
+0160   E2 98 83 71 0E 58 1A 00 00 00 58 2D 4F 62 6A 65    ...q.X....X-Obje
+0170   63 74 2D 53 79 73 6D 65 74 61 2D 45 63 2D 53 63    ct-Sysmeta-Ec-Sc
+0180   68 65 6D 65 71 0F 55 1A 6C 69 62 65 72 61 73 75    hemeq.U.liberasu
+0190   72 65 63 6F 64 65 5F 72 73 5F 76 61 6E 64 20 34    recode_rs_vand 4
+01A0   2B 32 71 10 55 0C 43 6F 6E 74 65 6E 74 2D 54 79    +2q.U.Content-Ty
+01B0   70 65 71 11 55 18 61 70 70 6C 69 63 61 74 69 6F    peq.U.applicatio
+01C0   6E 2F 6F 63 74 65 74 2D 73 74 72 65 61 6D 71 12    n/octet-streamq.
+01D0   58 20 00 00 00 58 2D 4F 62 6A 65 63 74 2D 53 79    X ...X-Object-Sy
+01E0   73 6D 65 74 61 2D 45 63 2D 53 65 67 6D 65 6E 74    smeta-Ec-Segment
+01F0   2D 53 69 7A 65 71 13 55 07 31 30 34 38 35 37 36    -Sizeq.U.1048576
+0200   71 14 75 2E                                        q.u.
+            ''',
+            'python_2.7.18_swift_2.23_replicated': '''
+0000   80 02 7D 71 01 28 55 0E 43 6F 6E 74 65 6E 74 2D    ..}q.(U.Content-
+0010   4C 65 6E 67 74 68 71 02 55 02 31 33 71 03 55 04    Lengthq.U.13q.U.
+0020   6E 61 6D 65 71 04 55 12 2F 41 55 54 48 5F 74 65    nameq.U./AUTH_te
+0030   73 74 2F E2 98 83 2F E2 98 83 71 05 55 0C 43 6F    st/.../...q.U.Co
+0040   6E 74 65 6E 74 2D 54 79 70 65 71 06 55 18 61 70    ntent-Typeq.U.ap
+0050   70 6C 69 63 61 74 69 6F 6E 2F 6F 63 74 65 74 2D    plication/octet-
+0060   73 74 72 65 61 6D 71 07 55 04 45 54 61 67 71 08    streamq.U.ETagq.
+0070   55 20 36 62 37 64 39 61 31 63 35 64 31 36 37 63    U 6b7d9a1c5d167c
+0080   63 35 30 30 33 37 66 32 39 66 32 39 30 62 62 33    c50037f29f290bb3
+0090   37 35 71 09 55 0B 58 2D 54 69 6D 65 73 74 61 6D    75q.U.X-Timestam
+00A0   70 71 0A 55 10 31 36 38 32 39 36 33 32 30 39 2E    pq.U.1682963209.
+00B0   38 32 32 37 32 71 0B 55 11 58 2D 4F 62 6A 65 63    82272q.U.X-Objec
+00C0   74 2D 4D 65 74 61 2D E2 98 83 71 0C 55 03 E2 98    t-Meta-...q.U...
+00D0   83 71 0D 55 13 58 2D 4F 62 6A 65 63 74 2D 4D 65    .q.U.X-Object-Me
+00E0   74 61 2D 4D 74 69 6D 65 71 0E 55 11 31 36 38 32    ta-Mtimeq.U.1682
+00F0   39 35 39 38 37 34 2E 37 35 36 32 30 35 71 0F 75    959874.756205q.u
+0100   2E                                                 .
+            ''',
+            'python_3.10.6_swift_2.23_replicated': '''
+0000   80 02 7D 71 00 28 63 5F 63 6F 64 65 63 73 0A 65    ..}q.(c_codecs.e
+0010   6E 63 6F 64 65 0A 71 01 58 0B 00 00 00 58 2D 54    ncode.q.X....X-T
+0020   69 6D 65 73 74 61 6D 70 71 02 58 06 00 00 00 6C    imestampq.X....l
+0030   61 74 69 6E 31 71 03 86 71 04 52 71 05 68 01 58    atin1q..q.Rq.h.X
+0040   10 00 00 00 31 36 38 32 39 36 33 30 31 37 2E 31    ....1682963017.1
+0050   30 34 37 32 71 06 68 03 86 71 07 52 71 08 68 01    0472q.h..q.Rq.h.
+0060   58 0C 00 00 00 43 6F 6E 74 65 6E 74 2D 54 79 70    X....Content-Typ
+0070   65 71 09 68 03 86 71 0A 52 71 0B 68 01 58 18 00    eq.h..q.Rq.h.X..
+0080   00 00 61 70 70 6C 69 63 61 74 69 6F 6E 2F 6F 63    ..application/oc
+0090   74 65 74 2D 73 74 72 65 61 6D 71 0C 68 03 86 71    tet-streamq.h..q
+00A0   0D 52 71 0E 68 01 58 0E 00 00 00 43 6F 6E 74 65    .Rq.h.X....Conte
+00B0   6E 74 2D 4C 65 6E 67 74 68 71 0F 68 03 86 71 10    nt-Lengthq.h..q.
+00C0   52 71 11 68 01 58 02 00 00 00 31 33 71 12 68 03    Rq.h.X....13q.h.
+00D0   86 71 13 52 71 14 68 01 58 04 00 00 00 45 54 61    .q.Rq.h.X....ETa
+00E0   67 71 15 68 03 86 71 16 52 71 17 68 01 58 20 00    gq.h..q.Rq.h.X .
+00F0   00 00 36 62 37 64 39 61 31 63 35 64 31 36 37 63    ..6b7d9a1c5d167c
+0100   63 35 30 30 33 37 66 32 39 66 32 39 30 62 62 33    c50037f29f290bb3
+0110   37 35 71 18 68 03 86 71 19 52 71 1A 68 01 58 13    75q.h..q.Rq.h.X.
+0120   00 00 00 58 2D 4F 62 6A 65 63 74 2D 4D 65 74 61    ...X-Object-Meta
+0130   2D 4D 74 69 6D 65 71 1B 68 03 86 71 1C 52 71 1D    -Mtimeq.h..q.Rq.
+0140   68 01 58 11 00 00 00 31 36 38 32 39 35 39 38 37    h.X....168295987
+0150   34 2E 37 35 36 32 30 35 71 1E 68 03 86 71 1F 52    4.756205q.h..q.R
+0160   71 20 68 01 58 1A 00 00 00 58 2D 4F 62 6A 65 63    q h.X....X-Objec
+0170   74 2D 4D 65 74 61 2D C3 83 C2 A2 C3 82 C2 98 C3    t-Meta-.........
+0180   82 C2 83 71 21 68 03 86 71 22 52 71 23 68 01 58    ...q!h..q"Rq#h.X
+0190   0C 00 00 00 C3 83 C2 A2 C3 82 C2 98 C3 82 C2 83    ................
+01A0   71 24 68 03 86 71 25 52 71 26 68 01 58 04 00 00    q$h..q%Rq&h.X...
+01B0   00 6E 61 6D 65 71 27 68 03 86 71 28 52 71 29 68    .nameq'h..q(Rq)h
+01C0   01 58 18 00 00 00 2F 41 55 54 48 5F 74 65 73 74    .X..../AUTH_test
+01D0   2F C3 A2 C2 98 C2 83 2F C3 A2 C2 98 C2 83 71 2A    /....../......q*
+01E0   68 03 86 71 2B 52 71 2C 75 2E                      h..q+Rq,u.
+            ''',
+            'python_2.7.18_swift_2.23_ec': '''
+0000   80 02 7D 71 01 28 55 0E 43 6F 6E 74 65 6E 74 2D    ..}q.(U.Content-
+0010   4C 65 6E 67 74 68 71 02 55 02 38 34 71 03 55 04    Lengthq.U.84q.U.
+0020   6E 61 6D 65 71 04 55 12 2F 41 55 54 48 5F 74 65    nameq.U./AUTH_te
+0030   73 74 2F E2 98 83 2F E2 98 83 71 05 55 1E 58 2D    st/.../...q.U.X-
+0040   4F 62 6A 65 63 74 2D 53 79 73 6D 65 74 61 2D 45    Object-Sysmeta-E
+0050   63 2D 46 72 61 67 2D 49 6E 64 65 78 55 01 35 55    c-Frag-IndexU.5U
+0060   0C 43 6F 6E 74 65 6E 74 2D 54 79 70 65 71 06 55    .Content-Typeq.U
+0070   18 61 70 70 6C 69 63 61 74 69 6F 6E 2F 6F 63 74    .application/oct
+0080   65 74 2D 73 74 72 65 61 6D 71 07 55 22 58 2D 4F    et-streamq.U"X-O
+0090   62 6A 65 63 74 2D 53 79 73 6D 65 74 61 2D 45 63    bject-Sysmeta-Ec
+00A0   2D 43 6F 6E 74 65 6E 74 2D 4C 65 6E 67 74 68 55    -Content-LengthU
+00B0   02 31 33 71 08 55 18 58 2D 4F 62 6A 65 63 74 2D    .13q.U.X-Object-
+00C0   53 79 73 6D 65 74 61 2D 45 63 2D 45 74 61 67 55    Sysmeta-Ec-EtagU
+00D0   20 36 62 37 64 39 61 31 63 35 64 31 36 37 63 63     6b7d9a1c5d167cc
+00E0   35 30 30 33 37 66 32 39 66 32 39 30 62 62 33 37    50037f29f290bb37
+00F0   35 71 09 55 04 45 54 61 67 71 0A 55 20 65 32 66    5q.U.ETagq.U e2f
+0100   64 34 33 30 65 61 66 37 32 32 33 63 32 35 30 33    d430eaf7223c2503
+0110   63 34 65 38 33 30 31 63 66 66 33 37 63 71 0B 55    c4e8301cff37cq.U
+0120   0B 58 2D 54 69 6D 65 73 74 61 6D 70 71 0C 55 10    .X-Timestampq.U.
+0130   31 36 38 32 39 36 33 31 33 30 2E 33 35 39 38 36    1682963130.35986
+0140   71 0D 55 11 58 2D 4F 62 6A 65 63 74 2D 4D 65 74    q.U.X-Object-Met
+0150   61 2D E2 98 83 71 0E 55 03 E2 98 83 71 0F 55 1A    a-...q.U....q.U.
+0160   58 2D 4F 62 6A 65 63 74 2D 53 79 73 6D 65 74 61    X-Object-Sysmeta
+0170   2D 45 63 2D 53 63 68 65 6D 65 55 1A 6C 69 62 65    -Ec-SchemeU.libe
+0180   72 61 73 75 72 65 63 6F 64 65 5F 72 73 5F 76 61    rasurecode_rs_va
+0190   6E 64 20 34 2B 32 71 10 55 13 58 2D 4F 62 6A 65    nd 4+2q.U.X-Obje
+01A0   63 74 2D 4D 65 74 61 2D 4D 74 69 6D 65 71 11 55    ct-Meta-Mtimeq.U
+01B0   11 31 36 38 32 39 35 39 38 37 34 2E 37 35 36 32    .1682959874.7562
+01C0   30 35 71 12 55 20 58 2D 4F 62 6A 65 63 74 2D 53    05q.U X-Object-S
+01D0   79 73 6D 65 74 61 2D 45 63 2D 53 65 67 6D 65 6E    ysmeta-Ec-Segmen
+01E0   74 2D 53 69 7A 65 55 07 31 30 34 38 35 37 36 71    t-SizeU.1048576q
+01F0   13 75 2E                                           .u.
+            ''',
+            'python_3.10.6_swift_2.23_ec': '''
+0000   80 02 7D 71 00 28 63 5F 63 6F 64 65 63 73 0A 65    ..}q.(c_codecs.e
+0010   6E 63 6F 64 65 0A 71 01 58 0B 00 00 00 58 2D 54    ncode.q.X....X-T
+0020   69 6D 65 73 74 61 6D 70 71 02 58 06 00 00 00 6C    imestampq.X....l
+0030   61 74 69 6E 31 71 03 86 71 04 52 71 05 68 01 58    atin1q..q.Rq.h.X
+0040   10 00 00 00 31 36 38 32 39 36 32 39 35 35 2E 33    ....1682962955.3
+0050   37 35 34 36 71 06 68 03 86 71 07 52 71 08 68 01    7546q.h..q.Rq.h.
+0060   58 0C 00 00 00 43 6F 6E 74 65 6E 74 2D 54 79 70    X....Content-Typ
+0070   65 71 09 68 03 86 71 0A 52 71 0B 68 01 58 18 00    eq.h..q.Rq.h.X..
+0080   00 00 61 70 70 6C 69 63 61 74 69 6F 6E 2F 6F 63    ..application/oc
+0090   74 65 74 2D 73 74 72 65 61 6D 71 0C 68 03 86 71    tet-streamq.h..q
+00A0   0D 52 71 0E 68 01 58 0E 00 00 00 43 6F 6E 74 65    .Rq.h.X....Conte
+00B0   6E 74 2D 4C 65 6E 67 74 68 71 0F 68 03 86 71 10    nt-Lengthq.h..q.
+00C0   52 71 11 68 01 58 02 00 00 00 38 34 71 12 68 03    Rq.h.X....84q.h.
+00D0   86 71 13 52 71 14 68 01 58 04 00 00 00 45 54 61    .q.Rq.h.X....ETa
+00E0   67 71 15 68 03 86 71 16 52 71 17 68 01 58 20 00    gq.h..q.Rq.h.X .
+00F0   00 00 65 32 66 64 34 33 30 65 61 66 37 32 32 33    ..e2fd430eaf7223
+0100   63 32 35 30 33 63 34 65 38 33 30 31 63 66 66 33    c2503c4e8301cff3
+0110   37 63 71 18 68 03 86 71 19 52 71 1A 68 01 58 13    7cq.h..q.Rq.h.X.
+0120   00 00 00 58 2D 4F 62 6A 65 63 74 2D 4D 65 74 61    ...X-Object-Meta
+0130   2D 4D 74 69 6D 65 71 1B 68 03 86 71 1C 52 71 1D    -Mtimeq.h..q.Rq.
+0140   68 01 58 11 00 00 00 31 36 38 32 39 35 39 38 37    h.X....168295987
+0150   34 2E 37 35 36 32 30 35 71 1E 68 03 86 71 1F 52    4.756205q.h..q.R
+0160   71 20 68 01 58 1A 00 00 00 58 2D 4F 62 6A 65 63    q h.X....X-Objec
+0170   74 2D 4D 65 74 61 2D C3 83 C2 A2 C3 82 C2 98 C3    t-Meta-.........
+0180   82 C2 83 71 21 68 03 86 71 22 52 71 23 68 01 58    ...q!h..q"Rq#h.X
+0190   0C 00 00 00 C3 83 C2 A2 C3 82 C2 98 C3 82 C2 83    ................
+01A0   71 24 68 03 86 71 25 52 71 26 68 01 58 18 00 00    q$h..q%Rq&h.X...
+01B0   00 58 2D 4F 62 6A 65 63 74 2D 53 79 73 6D 65 74    .X-Object-Sysmet
+01C0   61 2D 45 63 2D 45 74 61 67 71 27 68 03 86 71 28    a-Ec-Etagq'h..q(
+01D0   52 71 29 68 01 58 20 00 00 00 36 62 37 64 39 61    Rq)h.X ...6b7d9a
+01E0   31 63 35 64 31 36 37 63 63 35 30 30 33 37 66 32    1c5d167cc50037f2
+01F0   39 66 32 39 30 62 62 33 37 35 71 2A 68 03 86 71    9f290bb375q*h..q
+0200   2B 52 71 2C 68 01 58 22 00 00 00 58 2D 4F 62 6A    +Rq,h.X"...X-Obj
+0210   65 63 74 2D 53 79 73 6D 65 74 61 2D 45 63 2D 43    ect-Sysmeta-Ec-C
+0220   6F 6E 74 65 6E 74 2D 4C 65 6E 67 74 68 71 2D 68    ontent-Lengthq-h
+0230   03 86 71 2E 52 71 2F 68 01 58 02 00 00 00 31 33    ..q.Rq/h.X....13
+0240   71 30 68 03 86 71 31 52 71 32 68 01 58 1E 00 00    q0h..q1Rq2h.X...
+0250   00 58 2D 4F 62 6A 65 63 74 2D 53 79 73 6D 65 74    .X-Object-Sysmet
+0260   61 2D 45 63 2D 46 72 61 67 2D 49 6E 64 65 78 71    a-Ec-Frag-Indexq
+0270   33 68 03 86 71 34 52 71 35 68 01 58 01 00 00 00    3h..q4Rq5h.X....
+0280   35 71 36 68 03 86 71 37 52 71 38 68 01 58 1A 00    5q6h..q7Rq8h.X..
+0290   00 00 58 2D 4F 62 6A 65 63 74 2D 53 79 73 6D 65    ..X-Object-Sysme
+02A0   74 61 2D 45 63 2D 53 63 68 65 6D 65 71 39 68 03    ta-Ec-Schemeq9h.
+02B0   86 71 3A 52 71 3B 68 01 58 1A 00 00 00 6C 69 62    .q:Rq;h.X....lib
+02C0   65 72 61 73 75 72 65 63 6F 64 65 5F 72 73 5F 76    erasurecode_rs_v
+02D0   61 6E 64 20 34 2B 32 71 3C 68 03 86 71 3D 52 71    and 4+2q<h..q=Rq
+02E0   3E 68 01 58 20 00 00 00 58 2D 4F 62 6A 65 63 74    >h.X ...X-Object
+02F0   2D 53 79 73 6D 65 74 61 2D 45 63 2D 53 65 67 6D    -Sysmeta-Ec-Segm
+0300   65 6E 74 2D 53 69 7A 65 71 3F 68 03 86 71 40 52    ent-Sizeq?h..q@R
+0310   71 41 68 01 58 07 00 00 00 31 30 34 38 35 37 36    qAh.X....1048576
+0320   71 42 68 03 86 71 43 52 71 44 68 01 58 04 00 00    qBh..qCRqDh.X...
+0330   00 6E 61 6D 65 71 45 68 03 86 71 46 52 71 47 68    .nameqEh..qFRqGh
+0340   01 58 18 00 00 00 2F 41 55 54 48 5F 74 65 73 74    .X..../AUTH_test
+0350   2F C3 A2 C2 98 C2 83 2F C3 A2 C2 98 C2 83 71 48    /....../......qH
+0360   68 03 86 71 49 52 71 4A 75 2E                      h..qIRqJu.
+            ''',
+        }
+
+        def raw_xattr(output):
+            return binascii.unhexlify(''.join(
+                line[7:55] for line in output.split('\n')
+            ).replace(' ', ''))
+
+        path = os.path.join(self.testdir, str(uuid.uuid4()))
+        for case, xattr_output in cases.items():
+            try:
+                to_write = raw_xattr(xattr_output)
+                with open(path, 'wb') as fp:
+                    xattr.setxattr(
+                        fp.fileno(), 'user.swift.metadata', to_write)
+                with open(path, 'rb') as fd:
+                    actual = diskfile.read_metadata(fd)
+                # name should come out as native strings
+                expected_name = b'/AUTH_test/\xe2\x98\x83/\xe2\x98\x83'
+                if not six.PY2:
+                    expected_name = expected_name.decode('utf8')
+                self.assertEqual(actual['name'], expected_name)
+                # other meta will be WSGI strings, though
+                self.assertEqual(
+                    actual['X-Object-Meta-\xe2\x98\x83'], '\xe2\x98\x83')
+            except Exception:
+                print('Failure in %s' % case, file=sys.stderr)
+                raise
+
     def test_write_read_metadata(self):
         path = os.path.join(self.testdir, str(uuid.uuid4()))
         metadata = {'name': '/a/c/o',
@@ -597,29 +827,6 @@ class TestDiskFileModuleMethods(unittest.TestCase):
             diskfile.write_metadata(fd, metadata)
         check_metadata(as_native, str)
 
-        # mock the read path to check the write path encoded persisted metadata
-        with mock.patch.object(diskfile, '_decode_metadata', lambda x: x):
-            check_metadata(as_bytes, bytes)
-
-        # simulate a legacy diskfile that might have persisted
-        # (some) unicode metadata
-        with mock.patch.object(diskfile, '_encode_metadata', lambda x: x):
-            with open(path, 'wb') as fd:
-                diskfile.write_metadata(fd, metadata)
-        # sanity check: mock read path again to see that we did persist unicode
-        with mock.patch.object(diskfile, '_decode_metadata', lambda x: x):
-            with open(path, 'rb') as fd:
-                actual = diskfile.read_metadata(fd)
-                for k, v in actual.items():
-                    if isinstance(k, six.text_type) and \
-                            k == u'X-Object-Meta-Strange':
-                        self.assertIsInstance(v, six.text_type)
-                        break
-                else:
-                    self.fail('Did not find X-Object-Meta-Strange')
-        # check that read_metadata converts binary_type
-        check_metadata(as_native, str)
-
 
 @patch_policies
 class TestObjectAuditLocationGenerator(unittest.TestCase):