From 171c5381135bdfc6db154100ad5a9913955408a7 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 17 Oct 2013 08:44:25 +0900 Subject: refactoring. --- msgpack/fallback.py | 17 ----------------- 1 file changed, 17 deletions(-) (limited to 'msgpack/fallback.py') diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 8f9d646..3bf0489 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -60,23 +60,6 @@ TYPE_RAW = 3 DEFAULT_RECURSE_LIMIT=511 -def pack(o, stream, **kwargs): - """ - Pack object `o` and write it to `stream` - - See :class:`Packer` for options. - """ - packer = Packer(**kwargs) - stream.write(packer.pack(o)) - -def packb(o, **kwargs): - """ - Pack object `o` and return packed bytes - - See :class:`Packer` for options. - """ - return Packer(**kwargs).pack(o) - def unpack(stream, **kwargs): """ Unpack an object from `stream`. -- cgit v1.2.1 From 84f6b100190049a5d1bceb916208eeae2a3d2591 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 17 Oct 2013 08:52:59 +0900 Subject: Add bin type support to pure Python packer. --- msgpack/fallback.py | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) (limited to 'msgpack/fallback.py') diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 3bf0489..49a2bc5 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -429,11 +429,15 @@ class Packer(object): :param bool autoreset: Reset buffer after each pack and return it's content as `bytes`. (default: True). If set this to false, use `bytes()` to get content and `.reset()` to clear buffer. + :param bool use_bin_type: + Use bin type introduced in msgpack spec 2.0 for bytes. + It also enable str8 type for unicode. """ def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', - use_single_float=False, autoreset=True): + use_single_float=False, autoreset=True, use_bin_type=False): self._use_float = use_single_float self._autoreset = autoreset + self._use_bin_type = use_bin_type self._encoding = encoding self._unicode_errors = unicode_errors self._buffer = StringIO() @@ -473,6 +477,17 @@ class Packer(object): if -0x8000000000000000 <= obj < -0x80000000: return self._buffer.write(struct.pack(">Bq", 0xd3, obj)) raise PackValueError("Integer value out of range") + if self._use_bin_type and isinstance(obj, bytes): + n = len(obj) + if n <= 0xff: + self._buffer.write(struct.pack('>BB', 0xc4, n)) + elif n <= 0xffff: + self._buffer.write(struct.pack(">BH", 0xc5, n)) + elif n <= 0xffffffff: + self._buffer.write(struct.pack(">BI", 0xc6, n)) + else: + raise PackValueError("Bytes is too large") + return self._buffer.write(obj) if isinstance(obj, (Unicode, bytes)): if isinstance(obj, Unicode): if self._encoding is None: @@ -483,19 +498,20 @@ class Packer(object): n = len(obj) if n <= 0x1f: self._buffer.write(struct.pack('B', 0xa0 + n)) - return self._buffer.write(obj) - if n <= 0xffff: + elif self._use_bin_type and n <= 0xff: + self._buffer.write(struct.pack('>BB', 0xd9, n)) + elif n <= 0xffff: self._buffer.write(struct.pack(">BH", 0xda, n)) - return self._buffer.write(obj) - if n <= 0xffffffff: + elif n <= 0xffffffff: self._buffer.write(struct.pack(">BI", 0xdb, n)) - return self._buffer.write(obj) - raise PackValueError("String is too large") + else: + raise PackValueError("String is too large") + return self._buffer.write(obj) if isinstance(obj, float): if self._use_float: return self._buffer.write(struct.pack(">Bf", 0xca, obj)) return self._buffer.write(struct.pack(">Bd", 0xcb, obj)) - if isinstance(obj, list) or isinstance(obj, tuple): + if isinstance(obj, (list, tuple)): n = len(obj) self._fb_pack_array_header(n) for i in xrange(n): -- cgit v1.2.1 From 85eaff344b74092ccf076ba2465e9bdb1537409b Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 17 Oct 2013 09:15:19 +0900 Subject: Add bin type support for fallback Unpacker. --- msgpack/fallback.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) (limited to 'msgpack/fallback.py') diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 49a2bc5..901c6c8 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -57,6 +57,7 @@ TYPE_IMMEDIATE = 0 TYPE_ARRAY = 1 TYPE_MAP = 2 TYPE_RAW = 3 +TYPE_BIN = 4 DEFAULT_RECURSE_LIMIT=511 @@ -297,6 +298,10 @@ class Unpacker(object): obj = struct.unpack(">i", self._fb_read(4, write_bytes))[0] elif b == 0xd3: obj = struct.unpack(">q", self._fb_read(8, write_bytes))[0] + elif b == 0xd9: + n = struct.unpack("B", self._fb_read(1, write_bytes))[0] + obj = self._fb_read(n, write_bytes) + typ = TYPE_RAW elif b == 0xda: n = struct.unpack(">H", self._fb_read(2, write_bytes))[0] obj = self._fb_read(n, write_bytes) @@ -305,6 +310,18 @@ class Unpacker(object): n = struct.unpack(">I", self._fb_read(4, write_bytes))[0] obj = self._fb_read(n, write_bytes) typ = TYPE_RAW + elif b == 0xc4: + n = struct.unpack("B", self._fb_read(1, write_bytes))[0] + obj = self._fb_read(n, write_bytes) + typ = TYPE_BIN + elif b == 0xc5: + n = struct.unpack(">H", self._fb_read(2, write_bytes))[0] + obj = self._fb_read(n, write_bytes) + typ = TYPE_BIN + elif b == 0xc6: + n = struct.unpack(">I", self._fb_read(4, write_bytes))[0] + obj = self._fb_read(n, write_bytes) + typ = TYPE_BIN elif b == 0xdc: n = struct.unpack(">H", self._fb_read(2, write_bytes))[0] typ = TYPE_ARRAY @@ -373,6 +390,8 @@ class Unpacker(object): if self._encoding is not None: obj = obj.decode(self._encoding, self._unicode_errors) return obj + if typ == TYPE_BIN: + return obj assert typ == TYPE_IMMEDIATE return obj -- cgit v1.2.1 From d9439204c77a0ab58248c0b78237f16c4496647c Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 17 Oct 2013 11:29:36 +0900 Subject: Add ext type support to fallback.Unpacker. --- msgpack/fallback.py | 72 +++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 56 insertions(+), 16 deletions(-) (limited to 'msgpack/fallback.py') diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 901c6c8..074fcba 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -48,6 +48,9 @@ from msgpack.exceptions import ( PackValueError, ExtraData) +from msgpack import ExtType + + EX_SKIP = 0 EX_CONSTRUCT = 1 EX_READ_ARRAY_HEADER = 2 @@ -58,8 +61,9 @@ TYPE_ARRAY = 1 TYPE_MAP = 2 TYPE_RAW = 3 TYPE_BIN = 4 +TYPE_EXT = 5 -DEFAULT_RECURSE_LIMIT=511 +DEFAULT_RECURSE_LIMIT = 511 def unpack(stream, **kwargs): """ @@ -112,6 +116,9 @@ class Unpacker(object): should be callable and Unpacker calls it with a list of key-value pairs after deserializing a map. + `ext_hook` is callback for ext (User defined) type. It called with two + arguments: (code, bytes). default: `msgpack.ExtType` + `encoding` is the encoding used for decoding msgpack bytes. If it is None (default), msgpack bytes are deserialized to Python bytes. @@ -143,7 +150,8 @@ class Unpacker(object): def __init__(self, file_like=None, read_size=0, use_list=True, object_hook=None, object_pairs_hook=None, list_hook=None, - encoding=None, unicode_errors='strict', max_buffer_size=0): + encoding=None, unicode_errors='strict', max_buffer_size=0, + ext_hook=ExtType): if file_like is None: self._fb_feeding = True else: @@ -167,6 +175,7 @@ class Unpacker(object): self._list_hook = list_hook self._object_hook = object_hook self._object_pairs_hook = object_pairs_hook + self._ext_hook = ext_hook if list_hook is not None and not callable(list_hook): raise ValueError('`list_hook` is not callable') @@ -177,6 +186,8 @@ class Unpacker(object): if object_hook is not None and object_pairs_hook is not None: raise ValueError("object_pairs_hook and object_hook are mutually " "exclusive") + if not callable(ext_hook): + raise ValueError("`ext_hook` is not callable") def feed(self, next_bytes): if isinstance(next_bytes, array.array): @@ -278,6 +289,30 @@ class Unpacker(object): obj = False elif b == 0xc3: obj = True + elif b == 0xc4: + typ = TYPE_BIN + n = struct.unpack("B", self._fb_read(1, write_bytes))[0] + obj = self._fb_read(n, write_bytes) + elif b == 0xc5: + typ = TYPE_BIN + n = struct.unpack(">H", self._fb_read(2, write_bytes))[0] + obj = self._fb_read(n, write_bytes) + elif b == 0xc6: + typ = TYPE_BIN + n = struct.unpack(">I", self._fb_read(4, write_bytes))[0] + obj = self._fb_read(n, write_bytes) + elif b == 0xc7: # ext 8 + typ = TYPE_EXT + L, n = struct.unpack('Bb', self._fb_read(2, write_bytes)) + obj = self._fb_read(L, write_bytes) + elif b == 0xc8: # ext 16 + typ = TYPE_EXT + L, n = struct.unpack('>Hb', self._fb_read(3, write_bytes)) + obj = self._fb_read(L, write_bytes) + elif b == 0xc9: # ext 32 + typ = TYPE_EXT + L, n = struct.unpack('>Ib', self._fb_read(5, write_bytes)) + obj = self._fb_read(L, write_bytes) elif b == 0xca: obj = struct.unpack(">f", self._fb_read(4, write_bytes))[0] elif b == 0xcb: @@ -298,30 +333,33 @@ class Unpacker(object): obj = struct.unpack(">i", self._fb_read(4, write_bytes))[0] elif b == 0xd3: obj = struct.unpack(">q", self._fb_read(8, write_bytes))[0] + elif b == 0xd4: # fixext 1 + typ = TYPE_EXT + n, obj = struct.unpack('b1s', self._fb_read(2, write_bytes)) + elif b == 0xd5: # fixext 2 + typ = TYPE_EXT + n, obj = struct.unpack('b2s', self._fb_read(3, write_bytes)) + elif b == 0xd6: # fixext 4 + typ = TYPE_EXT + n, obj = struct.unpack('b4s', self._fb_read(5, write_bytes)) + elif b == 0xd7: # fixext 8 + typ = TYPE_EXT + n, obj = struct.unpack('b8s', self._fb_read(9, write_bytes)) + elif b == 0xd8: # fixext 16 + typ = TYPE_EXT + n, obj = struct.unpack('b16s', self._fb_read(17, write_bytes)) elif b == 0xd9: + typ = TYPE_RAW n = struct.unpack("B", self._fb_read(1, write_bytes))[0] obj = self._fb_read(n, write_bytes) - typ = TYPE_RAW elif b == 0xda: + typ = TYPE_RAW n = struct.unpack(">H", self._fb_read(2, write_bytes))[0] obj = self._fb_read(n, write_bytes) - typ = TYPE_RAW elif b == 0xdb: - n = struct.unpack(">I", self._fb_read(4, write_bytes))[0] - obj = self._fb_read(n, write_bytes) typ = TYPE_RAW - elif b == 0xc4: - n = struct.unpack("B", self._fb_read(1, write_bytes))[0] - obj = self._fb_read(n, write_bytes) - typ = TYPE_BIN - elif b == 0xc5: - n = struct.unpack(">H", self._fb_read(2, write_bytes))[0] - obj = self._fb_read(n, write_bytes) - typ = TYPE_BIN - elif b == 0xc6: n = struct.unpack(">I", self._fb_read(4, write_bytes))[0] obj = self._fb_read(n, write_bytes) - typ = TYPE_BIN elif b == 0xdc: n = struct.unpack(">H", self._fb_read(2, write_bytes))[0] typ = TYPE_ARRAY @@ -390,6 +428,8 @@ class Unpacker(object): if self._encoding is not None: obj = obj.decode(self._encoding, self._unicode_errors) return obj + if typ == TYPE_EXT: + return self._ext_hook(n, obj) if typ == TYPE_BIN: return obj assert typ == TYPE_IMMEDIATE -- cgit v1.2.1 From 7123341ca89a9a3afee8521cc16a1a419ea8871e Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 20 Oct 2013 14:34:36 +0900 Subject: code refactoring. --- msgpack/fallback.py | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) (limited to 'msgpack/fallback.py') diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 074fcba..dfaaa54 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -163,12 +163,10 @@ class Unpacker(object): self._fb_buf_o = 0 self._fb_buf_i = 0 self._fb_buf_n = 0 - self._max_buffer_size = (2**31-1 if max_buffer_size == 0 - else max_buffer_size) - self._read_size = (read_size if read_size != 0 - else min(self._max_buffer_size, 2048)) + self._max_buffer_size = max_buffer_size or 2**31-1 if read_size > self._max_buffer_size: raise ValueError("read_size must be smaller than max_buffer_size") + self._read_size = read_size or min(self._max_buffer_size, 2048) self._encoding = encoding self._unicode_errors = unicode_errors self._use_list = use_list @@ -234,26 +232,26 @@ class Unpacker(object): return b''.join(bufs) def _fb_read(self, n, write_bytes=None): - if (write_bytes is None and self._fb_buf_i < len(self._fb_buffers) - and self._fb_buf_o + n < len(self._fb_buffers[self._fb_buf_i])): + buffs = self._fb_buffers + if (write_bytes is None and self._fb_buf_i < len(buffs) and + self._fb_buf_o + n < len(buffs[self._fb_buf_i])): self._fb_buf_o += n - return self._fb_buffers[self._fb_buf_i][ - self._fb_buf_o-n:self._fb_buf_o] + return buffs[self._fb_buf_i][self._fb_buf_o - n:self._fb_buf_o] + ret = b'' while len(ret) != n: - if self._fb_buf_i == len(self._fb_buffers): + if self._fb_buf_i == len(buffs): if self._fb_feeding: break tmp = self.file_like.read(self._read_size) if not tmp: break - self._fb_buffers.append(tmp) + buffs.append(tmp) continue sliced = n - len(ret) - ret += self._fb_buffers[self._fb_buf_i][ - self._fb_buf_o:self._fb_buf_o + sliced] + ret += buffs[self._fb_buf_i][self._fb_buf_o:self._fb_buf_o + sliced] self._fb_buf_o += sliced - if self._fb_buf_o >= len(self._fb_buffers[self._fb_buf_i]): + if self._fb_buf_o >= len(buffs[self._fb_buf_i]): self._fb_buf_o = 0 self._fb_buf_i += 1 if len(ret) != n: @@ -410,10 +408,9 @@ class Unpacker(object): return if self._object_pairs_hook is not None: ret = self._object_pairs_hook( - (self._fb_unpack(EX_CONSTRUCT, write_bytes), - self._fb_unpack(EX_CONSTRUCT, write_bytes)) - for _ in xrange(n) - ) + (self._fb_unpack(EX_CONSTRUCT, write_bytes), + self._fb_unpack(EX_CONSTRUCT, write_bytes)) + for _ in xrange(n)) else: ret = {} for _ in xrange(n): -- cgit v1.2.1