diff options
| -rw-r--r-- | Makefile | 3 | ||||
| -rw-r--r-- | README.rst | 24 | ||||
| -rw-r--r-- | msgpack/_unpacker.pyx | 26 | ||||
| -rw-r--r-- | msgpack/fallback.py | 16 | ||||
| -rw-r--r-- | msgpack/unpack.h | 4 | ||||
| -rw-r--r-- | test/test_limits.py | 4 | ||||
| -rw-r--r-- | test/test_pack.py | 8 | ||||
| -rw-r--r-- | test/test_stricttype.py | 8 | ||||
| -rw-r--r-- | test/test_unpack.py | 2 |
9 files changed, 48 insertions, 47 deletions
@@ -18,7 +18,8 @@ serve-doc: all .PHONY: clean clean: rm -rf build - rm msgpack/*.so + rm -f msgpack/_packer.cpp + rm -f msgpack/_unpacker.cpp rm -rf msgpack/__pycache__ rm -rf test/__pycache__ @@ -47,9 +47,9 @@ In case of packer, use UTF-8 always. Storing other than UTF-8 is not recommende For backward compatibility, you can use ``use_bin_type=False`` and pack ``bytes`` object into msgpack raw type. -In case of unpacker, there is new ``raw_as_bytes`` option. It is ``True`` by default +In case of unpacker, there is new ``raw`` option. It is ``True`` by default for backward compatibility, but it is changed to ``False`` in near future. -You can use ``raw_as_bytes=False`` instead of ``encoding='utf-8'``. +You can use ``raw=False`` instead of ``encoding='utf-8'``. Planned backward incompatible changes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -58,14 +58,14 @@ When msgpack 1.0, I planning these breaking changes: * packer and unpacker: Remove ``encoding`` and ``unicode_errors`` option. * packer: Change default of ``use_bin_type`` option from False to True. -* unpacker: Change default of ``raw_as_bytes`` option from True to False. +* unpacker: Change default of ``raw`` option from True to False. * unpacker: Reduce all ``max_xxx_len`` options for typical usage. * unpacker: Remove ``write_bytes`` option from all methods. To avoid these breaking changes breaks your application, please: * Don't use deprecated options. -* Pass ``use_bin_type`` and ``raw_as_bytes`` options explicitly. +* Pass ``use_bin_type`` and ``raw`` options explicitly. * If your application handle large (>1MB) data, specify ``max_xxx_len`` options too. @@ -113,14 +113,14 @@ msgpack provides ``dumps`` and ``loads`` as an alias for compatibility with >>> import msgpack >>> msgpack.packb([1, 2, 3], use_bin_type=True) '\x93\x01\x02\x03' - >>> msgpack.unpackb(_, raw_as_bytes=False) + >>> msgpack.unpackb(_, raw=False) [1, 2, 3] ``unpack`` unpacks msgpack's array to Python's list, but can also unpack to tuple: .. code-block:: pycon - >>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False, raw_as_bytes=False) + >>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False, raw=False) (1, 2, 3) You should always specify the ``use_list`` keyword argument for backward compatibility. @@ -146,7 +146,7 @@ stream (or from bytes provided through its ``feed`` method). buf.seek(0) - unpacker = msgpack.Unpacker(buf, raw_as_bytes=False) + unpacker = msgpack.Unpacker(buf, raw=False) for unpacked in unpacker: print(unpacked) @@ -179,7 +179,7 @@ It is also possible to pack/unpack custom data types. Here is an example for packed_dict = msgpack.packb(useful_dict, default=encode_datetime, use_bin_type=True) - this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime, raw_as_bytes=False) + this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime, raw=False) ``Unpacker``'s ``object_hook`` callback receives a dict; the ``object_pairs_hook`` callback may instead be used to receive a list of @@ -209,7 +209,7 @@ It is also possible to pack/unpack custom data types using the **ext** type. ... >>> data = array.array('d', [1.2, 3.4]) >>> packed = msgpack.packb(data, default=default, use_bin_type=True) - >>> unpacked = msgpack.unpackb(packed, ext_hook=ext_hook, raw_as_bytes=False) + >>> unpacked = msgpack.unpackb(packed, ext_hook=ext_hook, raw=False) >>> data == unpacked True @@ -257,7 +257,7 @@ For backward compatibility reasons, msgpack-python will still default all strings to byte strings, unless you specify the ``use_bin_type=True`` option in the packer. If you do so, it will use a non-standard type called **bin** to serialize byte arrays, and **raw** becomes to mean **str**. If you want to -distinguish **bin** and **raw** in the unpacker, specify ``raw_as_bytes=False``. +distinguish **bin** and **raw** in the unpacker, specify ``raw=False``. Note that Python 2 defaults to byte-arrays over Unicode strings: @@ -267,7 +267,7 @@ Note that Python 2 defaults to byte-arrays over Unicode strings: >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'])) ['spam', 'eggs'] >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=True), - raw_as_bytes=False) + raw=False) ['spam', u'eggs'] This is the same code in Python 3 (same behaviour, but Python 3 has a @@ -279,7 +279,7 @@ different default): >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'])) [b'spam', b'eggs'] >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=True), - raw_as_bytes=False) + raw=False) [b'spam', 'eggs'] diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index b796d04..806be4f 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -43,7 +43,7 @@ from msgpack import ExtType cdef extern from "unpack.h": ctypedef struct msgpack_user: bint use_list - bint raw_as_bytes + bint raw bint has_pairs_hook # call object_hook with k-v pairs PyObject* object_hook PyObject* list_hook @@ -74,14 +74,14 @@ cdef extern from "unpack.h": cdef inline init_ctx(unpack_context *ctx, object object_hook, object object_pairs_hook, object list_hook, object ext_hook, - bint use_list, bint raw_as_bytes, + bint use_list, bint raw, char* encoding, char* unicode_errors, Py_ssize_t max_str_len, Py_ssize_t max_bin_len, Py_ssize_t max_array_len, Py_ssize_t max_map_len, Py_ssize_t max_ext_len): unpack_init(ctx) ctx.user.use_list = use_list - ctx.user.raw_as_bytes = raw_as_bytes + ctx.user.raw = raw ctx.user.object_hook = ctx.user.list_hook = <PyObject*>NULL ctx.user.max_str_len = max_str_len ctx.user.max_bin_len = max_bin_len @@ -158,7 +158,7 @@ cdef inline int get_data_from_buffer(object obj, return 1 def unpackb(object packed, object object_hook=None, object list_hook=None, - bint use_list=True, bint raw_as_bytes=True, + bint use_list=True, bint raw=True, encoding=None, unicode_errors="strict", object_pairs_hook=None, ext_hook=ExtType, Py_ssize_t max_str_len=2147483647, # 2**32-1 @@ -185,7 +185,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, cdef int new_protocol = 0 if encoding is not None: - PyErr_WarnEx(PendingDeprecationWarning, "encoding is deprecated, Use raw_as_bytes=False instead.", 1) + PyErr_WarnEx(PendingDeprecationWarning, "encoding is deprecated, Use raw=False instead.", 1) if isinstance(encoding, unicode): encoding = encoding.encode('ascii') elif not isinstance(encoding, bytes): @@ -203,7 +203,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, get_data_from_buffer(packed, &view, &buf, &buf_len, &new_protocol) try: init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, ext_hook, - use_list, raw_as_bytes, cenc, cerr, + use_list, raw, cenc, cerr, max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len) ret = unpack_construct(&ctx, buf, buf_len, &off) finally: @@ -261,7 +261,7 @@ cdef class Unpacker(object): If true, unpack msgpack array to Python list. Otherwise, unpack to Python tuple. (default: True) - :param bool raw_as_bytes: + :param bool raw: If true, unpack msgpack raw to Python bytes (default). Otherwise, unpack to Python str (or unicode on Python 2) by decoding with UTF-8 encoding (recommended). @@ -299,7 +299,7 @@ cdef class Unpacker(object): Limits max length of map. (default: 2**31-1) :param str encoding: - Deprecated, use raw_as_bytes instead. + Deprecated, use raw instead. Encoding used for decoding msgpack raw. If it is None (default), msgpack raw is deserialized to Python bytes. @@ -310,13 +310,13 @@ cdef class Unpacker(object): Example of streaming deserialize from file-like object:: - unpacker = Unpacker(file_like, raw_as_bytes=False) + unpacker = Unpacker(file_like, raw=False) for o in unpacker: process(o) Example of streaming deserialize from socket:: - unpacker = Unpacker(raw_as_bytes=False) + unpacker = Unpacker(raw=False) while True: buf = sock.recv(1024**2) if not buf: @@ -345,7 +345,7 @@ cdef class Unpacker(object): self.buf = NULL def __init__(self, file_like=None, Py_ssize_t read_size=0, - bint use_list=True, bint raw_as_bytes=True, + bint use_list=True, bint raw=True, object object_hook=None, object object_pairs_hook=None, object list_hook=None, encoding=None, unicode_errors='strict', int max_buffer_size=0, object ext_hook=ExtType, @@ -384,7 +384,7 @@ cdef class Unpacker(object): self.stream_offset = 0 if encoding is not None: - PyErr_WarnEx(PendingDeprecationWarning, "encoding is deprecated, Use raw_as_bytes=False instead.", 1) + PyErr_WarnEx(PendingDeprecationWarning, "encoding is deprecated, Use raw=False instead.", 1) if isinstance(encoding, unicode): self.encoding = encoding.encode('ascii') elif isinstance(encoding, bytes): @@ -404,7 +404,7 @@ cdef class Unpacker(object): cerr = PyBytes_AsString(self.unicode_errors) init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, - ext_hook, use_list, raw_as_bytes, cenc, cerr, + ext_hook, use_list, raw, cenc, cerr, max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 675ee8a..bf6c9a6 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -145,7 +145,7 @@ class Unpacker(object): If true, unpack msgpack array to Python list. Otherwise, unpack to Python tuple. (default: True) - :param bool raw_as_bytes: + :param bool raw: If true, unpack msgpack raw to Python bytes (default). Otherwise, unpack to Python str (or unicode on Python 2) by decoding with UTF-8 encoding (recommended). @@ -193,13 +193,13 @@ class Unpacker(object): example of streaming deserialize from file-like object:: - unpacker = Unpacker(file_like, raw_as_bytes=False) + unpacker = Unpacker(file_like, raw=False) for o in unpacker: process(o) example of streaming deserialize from socket:: - unpacker = Unpacker(raw_as_bytes=False) + unpacker = Unpacker(raw=False) while True: buf = sock.recv(1024**2) if not buf: @@ -209,7 +209,7 @@ class Unpacker(object): process(o) """ - def __init__(self, file_like=None, read_size=0, use_list=True, raw_as_bytes=True, + def __init__(self, file_like=None, read_size=0, use_list=True, raw=True, object_hook=None, object_pairs_hook=None, list_hook=None, encoding=None, unicode_errors=None, max_buffer_size=0, ext_hook=ExtType, @@ -221,7 +221,7 @@ class Unpacker(object): if encoding is not None: warnings.warn( - "encoding is deprecated, Use raw_as_bytes=False instead.", + "encoding is deprecated, Use raw=False instead.", PendingDeprecationWarning) if unicode_errors is not None: @@ -257,7 +257,7 @@ class Unpacker(object): if read_size > self._max_buffer_size: raise ValueError("read_size must be smaller than max_buffer_size") self._read_size = read_size or min(self._max_buffer_size, 16*1024) - self._raw_as_bytes = bool(raw_as_bytes) + self._raw = bool(raw) self._encoding = encoding self._unicode_errors = unicode_errors self._use_list = use_list @@ -606,7 +606,7 @@ class Unpacker(object): if typ == TYPE_RAW: if self._encoding is not None: obj = obj.decode(self._encoding, self._unicode_errors) - elif self._raw_as_bytes: + elif self._raw: obj = bytes(obj) else: obj = obj.decode('utf_8') @@ -715,7 +715,7 @@ class Packer(object): encoding = 'utf_8' else: warnings.warn( - "encoding is deprecated, Use raw_as_bytes=False instead.", + "encoding is deprecated, Use raw=False instead.", PendingDeprecationWarning) if unicode_errors is None: diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 8c2fc46..d7b5e00 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -21,7 +21,7 @@ typedef struct unpack_user { bool use_list; - bool raw_as_bytes; + bool raw; bool has_pairs_hook; PyObject *object_hook; PyObject *list_hook; @@ -229,7 +229,7 @@ static inline int unpack_callback_raw(unpack_user* u, const char* b, const char* if (u->encoding) { py = PyUnicode_Decode(p, l, u->encoding, u->unicode_errors); - } else if (u->raw_as_bytes) { + } else if (u->raw) { py = PyBytes_FromStringAndSize(p, l); } else { py = PyUnicode_DecodeUTF8(p, l, NULL); diff --git a/test/test_limits.py b/test/test_limits.py index 3febc30..74e48c1 100644 --- a/test/test_limits.py +++ b/test/test_limits.py @@ -39,11 +39,11 @@ def test_max_str_len(): d = 'x' * 3 packed = packb(d) - unpacker = Unpacker(max_str_len=3, raw_as_bytes=False) + unpacker = Unpacker(max_str_len=3, raw=False) unpacker.feed(packed) assert unpacker.unpack() == d - unpacker = Unpacker(max_str_len=2, raw_as_bytes=False) + unpacker = Unpacker(max_str_len=2, raw=False) with pytest.raises(UnpackValueError): unpacker.feed(packed) unpacker.unpack() diff --git a/test/test_pack.py b/test/test_pack.py index 29f5887..b447f9c 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -31,11 +31,11 @@ def testPack(): def testPackUnicode(): test_data = ["", "abcd", ["defgh"], "Русский текст"] for td in test_data: - re = unpackb(packb(td), use_list=1, raw_as_bytes=False) + re = unpackb(packb(td), use_list=1, raw=False) assert re == td packer = Packer() data = packer.pack(td) - re = Unpacker(BytesIO(data), raw_as_bytes=False, use_list=1).unpack() + re = Unpacker(BytesIO(data), raw=False, use_list=1).unpack() assert re == td def testPackUTF32(): # deprecated @@ -72,14 +72,14 @@ def testIgnoreUnicodeErrors(): # deprecated def testStrictUnicodeUnpack(): with raises(UnicodeDecodeError): - unpackb(packb(b'abc\xeddef'), raw_as_bytes=False, use_list=1) + unpackb(packb(b'abc\xeddef'), raw=False, use_list=1) def testStrictUnicodePack(): # deprecated with raises(UnicodeEncodeError): packb("abc\xeddef", encoding='ascii', unicode_errors='strict') def testIgnoreErrorsPack(): # deprecated - re = unpackb(packb("abcФФФdef", encoding='ascii', unicode_errors='ignore'), raw_as_bytes=False, use_list=1) + re = unpackb(packb("abcФФФdef", encoding='ascii', unicode_errors='ignore'), raw=False, use_list=1) assert re == "abcdef" def testDecodeBinary(): diff --git a/test/test_stricttype.py b/test/test_stricttype.py index 13239f1..87e7c1c 100644 --- a/test/test_stricttype.py +++ b/test/test_stricttype.py @@ -11,7 +11,7 @@ def test_namedtuple(): return dict(o._asdict()) raise TypeError('Unsupported type %s' % (type(o),)) packed = packb(T(1, 42), strict_types=True, use_bin_type=True, default=default) - unpacked = unpackb(packed, raw_as_bytes=False) + unpacked = unpackb(packed, raw=False) assert unpacked == {'foo': 1, 'bar': 42} @@ -32,7 +32,7 @@ def test_tuple(): return o data = packb(t, strict_types=True, use_bin_type=True, default=default) - expected = unpackb(data, raw_as_bytes=False, object_hook=convert) + expected = unpackb(data, raw=False, object_hook=convert) assert expected == t @@ -53,10 +53,10 @@ def test_tuple_ext(): def convert(code, payload): if code == MSGPACK_EXT_TYPE_TUPLE: # Unpack and convert to tuple - return tuple(unpackb(payload, raw_as_bytes=False, ext_hook=convert)) + return tuple(unpackb(payload, raw=False, ext_hook=convert)) raise ValueError('Unknown Ext code {}'.format(code)) data = packb(t, strict_types=True, use_bin_type=True, default=default) - expected = unpackb(data, raw_as_bytes=False, ext_hook=convert) + expected = unpackb(data, raw=False, ext_hook=convert) assert expected == t diff --git a/test/test_unpack.py b/test/test_unpack.py index 143f999..00a1061 100644 --- a/test/test_unpack.py +++ b/test/test_unpack.py @@ -48,7 +48,7 @@ def test_unpacker_ext_hook(): def __init__(self): super(MyUnpacker, self).__init__( - ext_hook=self._hook, raw_as_bytes=False) + ext_hook=self._hook, raw=False) def _hook(self, code, data): if code == 1: |