diff options
author | INADA Naoki <inada-n@klab.com> | 2014-06-26 21:41:50 +0900 |
---|---|---|
committer | INADA Naoki <inada-n@klab.com> | 2014-06-26 21:41:50 +0900 |
commit | c3cc8a96460c94cdde8eee6e83df6cb661d42018 (patch) | |
tree | 5d53902deaa4a7a439b47c21f029744ce5909f1c | |
parent | fc546ec4cd135467aa3ee2e33061f17a6549ae98 (diff) | |
download | msgpack-python-c3cc8a96460c94cdde8eee6e83df6cb661d42018.tar.gz |
Implement max_xxx_type for fallback.
-rw-r--r-- | msgpack/_unpacker.pyx | 3 | ||||
-rw-r--r-- | msgpack/fallback.py | 127 |
2 files changed, 99 insertions, 31 deletions
diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index f69be91..a9770d4 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -163,8 +163,7 @@ def unpack(object stream, object object_hook=None, object list_hook=None, cdef class Unpacker(object): - """ - Streaming unpacker. + """Streaming unpacker. arguments: diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 49323e6..f50adde 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -99,62 +99,84 @@ def unpackb(packed, **kwargs): class Unpacker(object): - """ - Streaming unpacker. + """Streaming unpacker. + + arguments: - `file_like` is a file-like object having a `.read(n)` method. - When `Unpacker` is initialized with a `file_like`, `.feed()` is not - usable. + :param file_like: + File-like object having `.read(n)` method. + If specified, unpacker reads serialized data from it and :meth:`feed()` is not usable. - `read_size` is used for `file_like.read(read_size)`. + :param int read_size: + Used as `file_like.read(read_size)`. (default: `min(1024**2, max_buffer_size)`) - If `use_list` is True (default), msgpack lists are deserialized to Python - lists. Otherwise they are deserialized to tuples. + :param bool use_list: + If true, unpack msgpack array to Python list. + Otherwise, unpack to Python tuple. (default: True) - `object_hook` is the same as in simplejson. If it is not None, it should - be callable and Unpacker calls it with a dict argument after deserializing - a map. + :param callable object_hook: + When specified, it should be callable. + Unpacker calls it with a dict argument after unpacking msgpack map. + (See also simplejson) - `object_pairs_hook` is the same as in simplejson. If it is not None, it - should be callable and Unpacker calls it with a list of key-value pairs - after deserializing a map. + :param callable object_pairs_hook: + When specified, it should be callable. + Unpacker calls it with a list of key-value pairs after unpacking msgpack map. + (See also simplejson) - `ext_hook` is callback for ext (User defined) type. It called with two - arguments: (code, bytes). default: `msgpack.ExtType` + :param str encoding: + Encoding used for decoding msgpack raw. + If it is None (default), msgpack raw is deserialized to Python bytes. - `encoding` is the encoding used for decoding msgpack bytes. If it is - None (default), msgpack bytes are deserialized to Python bytes. + :param str unicode_errors: + Used for decoding msgpack raw with *encoding*. + (default: `'strict'`) - `unicode_errors` is used for decoding bytes. + :param int max_buffer_size: + Limits size of data waiting unpacked. 0 means system's INT_MAX (default). + Raises `BufferFull` exception when it is insufficient. + You shoud set this parameter when unpacking data from untrasted source. - `max_buffer_size` limits the buffer size. 0 means INT_MAX (default). + :param int max_str_len: + Limits max length of str. (default: 2**31-1) - Raises `BufferFull` exception when it is unsufficient. + :param int max_bin_len: + Limits max length of bin. (default: 2**31-1) - You should set this parameter when unpacking data from an untrustred source. + :param int max_array_len: + Limits max length of array. (default: 2**31-1) - example of streaming deserialization from file-like object:: + :param int max_map_len: + Limits max length of map. (default: 2**31-1) + + + example of streaming deserialize from file-like object:: unpacker = Unpacker(file_like) for o in unpacker: - do_something(o) + process(o) - example of streaming deserialization from socket:: + example of streaming deserialize from socket:: unpacker = Unpacker() - while 1: - buf = sock.recv(1024*2) + while True: + buf = sock.recv(1024**2) if not buf: break unpacker.feed(buf) for o in unpacker: - do_something(o) + process(o) """ def __init__(self, file_like=None, read_size=0, use_list=True, object_hook=None, object_pairs_hook=None, list_hook=None, encoding=None, unicode_errors='strict', max_buffer_size=0, - ext_hook=ExtType): + ext_hook=ExtType, + max_str_len=2147483647, # 2**32-1 + max_bin_len=2147483647, + max_array_len=2147483647, + max_map_len=2147483647, + max_ext_len=2147483647): if file_like is None: self._fb_feeding = True else: @@ -177,6 +199,11 @@ class Unpacker(object): self._object_hook = object_hook self._object_pairs_hook = object_pairs_hook self._ext_hook = ext_hook + self._max_str_len = max_str_len + self._max_bin_len = max_bin_len + self._max_array_len = max_array_len + self._max_map_len = max_map_len + self._max_ext_len = max_ext_len if list_hook is not None and not callable(list_hook): raise TypeError('`list_hook` is not callable') @@ -280,12 +307,18 @@ class Unpacker(object): n = b & 0b00011111 obj = self._fb_read(n, write_bytes) typ = TYPE_RAW + if n > self._max_str_len: + raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) elif b & 0b11110000 == 0b10010000: n = b & 0b00001111 typ = TYPE_ARRAY + if n > self._max_array_len: + raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) elif b & 0b11110000 == 0b10000000: n = b & 0b00001111 typ = TYPE_MAP + if n > self._max_map_len: + raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) elif b == 0xc0: obj = None elif b == 0xc2: @@ -295,26 +328,38 @@ class Unpacker(object): elif b == 0xc4: typ = TYPE_BIN n = struct.unpack("B", self._fb_read(1, write_bytes))[0] + if n > self._max_bin_len: + raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) obj = self._fb_read(n, write_bytes) elif b == 0xc5: typ = TYPE_BIN n = struct.unpack(">H", self._fb_read(2, write_bytes))[0] + if n > self._max_bin_len: + raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) obj = self._fb_read(n, write_bytes) elif b == 0xc6: typ = TYPE_BIN n = struct.unpack(">I", self._fb_read(4, write_bytes))[0] + if n > self._max_bin_len: + raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) obj = self._fb_read(n, write_bytes) elif b == 0xc7: # ext 8 typ = TYPE_EXT L, n = struct.unpack('Bb', self._fb_read(2, write_bytes)) + if L > self._max_ext_len: + raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) obj = self._fb_read(L, write_bytes) elif b == 0xc8: # ext 16 typ = TYPE_EXT L, n = struct.unpack('>Hb', self._fb_read(3, write_bytes)) + if L > self._max_ext_len: + raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) obj = self._fb_read(L, write_bytes) elif b == 0xc9: # ext 32 typ = TYPE_EXT L, n = struct.unpack('>Ib', self._fb_read(5, write_bytes)) + if L > self._max_ext_len: + raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) obj = self._fb_read(L, write_bytes) elif b == 0xca: obj = struct.unpack(">f", self._fb_read(4, write_bytes))[0] @@ -338,42 +383,66 @@ class Unpacker(object): obj = struct.unpack(">q", self._fb_read(8, write_bytes))[0] elif b == 0xd4: # fixext 1 typ = TYPE_EXT + if self._max_ext_len < 1: + raise ValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len)) n, obj = struct.unpack('b1s', self._fb_read(2, write_bytes)) elif b == 0xd5: # fixext 2 typ = TYPE_EXT + if self._max_ext_len < 2: + raise ValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len)) n, obj = struct.unpack('b2s', self._fb_read(3, write_bytes)) elif b == 0xd6: # fixext 4 typ = TYPE_EXT + if self._max_ext_len < 4: + raise ValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len)) n, obj = struct.unpack('b4s', self._fb_read(5, write_bytes)) elif b == 0xd7: # fixext 8 typ = TYPE_EXT + if self._max_ext_len < 8: + raise ValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len)) n, obj = struct.unpack('b8s', self._fb_read(9, write_bytes)) elif b == 0xd8: # fixext 16 typ = TYPE_EXT + if self._max_ext_len < 16: + raise ValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len)) n, obj = struct.unpack('b16s', self._fb_read(17, write_bytes)) elif b == 0xd9: typ = TYPE_RAW n = struct.unpack("B", self._fb_read(1, write_bytes))[0] + if n > self._max_str_len: + raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) obj = self._fb_read(n, write_bytes) elif b == 0xda: typ = TYPE_RAW n = struct.unpack(">H", self._fb_read(2, write_bytes))[0] + if n > self._max_str_len: + raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) obj = self._fb_read(n, write_bytes) elif b == 0xdb: typ = TYPE_RAW n = struct.unpack(">I", self._fb_read(4, write_bytes))[0] + if n > self._max_str_len: + raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) obj = self._fb_read(n, write_bytes) elif b == 0xdc: n = struct.unpack(">H", self._fb_read(2, write_bytes))[0] + if n > self._max_array_len: + raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) typ = TYPE_ARRAY elif b == 0xdd: n = struct.unpack(">I", self._fb_read(4, write_bytes))[0] + if n > self._max_array_len: + raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) typ = TYPE_ARRAY elif b == 0xde: n = struct.unpack(">H", self._fb_read(2, write_bytes))[0] + if n > self._max_map_len: + raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) typ = TYPE_MAP elif b == 0xdf: n = struct.unpack(">I", self._fb_read(4, write_bytes))[0] + if n > self._max_map_len: + raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) typ = TYPE_MAP else: raise UnpackValueError("Unknown header: 0x%x" % b) |