From c356035a576c38db5ca232ede07b291087f1b8b2 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 9 Dec 2019 17:03:12 +0900 Subject: Unpacker: Change max_buffer_size to 100MiB (#391) --- msgpack/_unpacker.pyx | 34 ++++++++++++++++++---------------- msgpack/fallback.py | 33 ++++++++++++++++++--------------- 2 files changed, 36 insertions(+), 31 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 53ecf86..0ff633b 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -234,27 +234,28 @@ cdef class Unpacker(object): (See also simplejson) :param int max_buffer_size: - Limits size of data waiting unpacked. 0 means system's INT_MAX (default). + Limits size of data waiting unpacked. 0 means system's INT_MAX. + The default value is 100*1024*1024 (100MiB). Raises `BufferFull` exception when it is insufficient. You should set this parameter when unpacking data from untrusted source. :param int max_str_len: Deprecated, use *max_buffer_size* instead. - Limits max length of str. (default: max_buffer_size or 1024*1024) + Limits max length of str. (default: max_buffer_size) :param int max_bin_len: Deprecated, use *max_buffer_size* instead. - Limits max length of bin. (default: max_buffer_size or 1024*1024) + Limits max length of bin. (default: max_buffer_size) :param int max_array_len: - Limits max length of array. (default: max_buffer_size or 128*1024) + Limits max length of array. (default: max_buffer_size) :param int max_map_len: - Limits max length of map. (default: max_buffer_size//2 or 32*1024) + Limits max length of map. (default: max_buffer_size//2) :param int max_ext_len: Deprecated, use *max_buffer_size* instead. - Limits max size of ext type. (default: max_buffer_size or 1024*1024) + Limits max size of ext type. (default: max_buffer_size) :param str unicode_errors: Error handler used for decoding str type. (default: `'strict'`) @@ -262,13 +263,13 @@ cdef class Unpacker(object): Example of streaming deserialize from file-like object:: - unpacker = Unpacker(file_like, max_buffer_size=10*1024*1024) + unpacker = Unpacker(file_like) for o in unpacker: process(o) Example of streaming deserialize from socket:: - unpacker = Unpacker(max_buffer_size=10*1024*1024) + unpacker = Unpacker() while True: buf = sock.recv(1024**2) if not buf: @@ -305,7 +306,7 @@ cdef class Unpacker(object): def __init__(self, file_like=None, *, Py_ssize_t read_size=0, bint use_list=True, bint raw=False, bint strict_map_key=True, object object_hook=None, object object_pairs_hook=None, object list_hook=None, - unicode_errors=None, Py_ssize_t max_buffer_size=0, + unicode_errors=None, Py_ssize_t max_buffer_size=100*1024*1024, object ext_hook=ExtType, Py_ssize_t max_str_len=-1, Py_ssize_t max_bin_len=-1, @@ -325,23 +326,24 @@ cdef class Unpacker(object): if not PyCallable_Check(self.file_like_read): raise TypeError("`file_like.read` must be a callable.") + if not max_buffer_size: + max_buffer_size = INT_MAX if max_str_len == -1: - max_str_len = max_buffer_size or 1024*1024 + max_str_len = max_buffer_size if max_bin_len == -1: - max_bin_len = max_buffer_size or 1024*1024 + max_bin_len = max_buffer_size if max_array_len == -1: - max_array_len = max_buffer_size or 128*1024 + max_array_len = max_buffer_size if max_map_len == -1: - max_map_len = max_buffer_size//2 or 32*1024 + max_map_len = max_buffer_size//2 if max_ext_len == -1: - max_ext_len = max_buffer_size or 1024*1024 + max_ext_len = max_buffer_size - if not max_buffer_size: - max_buffer_size = INT_MAX if read_size > max_buffer_size: raise ValueError("read_size should be less or equal to max_buffer_size") if not read_size: read_size = min(max_buffer_size, 1024**2) + self.max_buffer_size = max_buffer_size self.read_size = read_size self.buf = PyMem_Malloc(read_size) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 3704f9d..f6ba424 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -193,39 +193,40 @@ class Unpacker(object): contains invalid UTF-8 string. :param int max_buffer_size: - Limits size of data waiting unpacked. 0 means system's INT_MAX (default). + Limits size of data waiting unpacked. 0 means 2**32-1. + The default value is 100*1024*1024 (100MiB). Raises `BufferFull` exception when it is insufficient. You should set this parameter when unpacking data from untrusted source. :param int max_str_len: Deprecated, use *max_buffer_size* instead. - Limits max length of str. (default: max_buffer_size or 1024*1024) + Limits max length of str. (default: max_buffer_size) :param int max_bin_len: Deprecated, use *max_buffer_size* instead. - Limits max length of bin. (default: max_buffer_size or 1024*1024) + Limits max length of bin. (default: max_buffer_size) :param int max_array_len: Limits max length of array. - (default: max_buffer_size or 128*1024) + (default: max_buffer_size) :param int max_map_len: Limits max length of map. - (default: max_buffer_size//2 or 32*1024) + (default: max_buffer_size//2) :param int max_ext_len: Deprecated, use *max_buffer_size* instead. - Limits max size of ext type. (default: max_buffer_size or 1024*1024) + Limits max size of ext type. (default: max_buffer_size) Example of streaming deserialize from file-like object:: - unpacker = Unpacker(file_like, max_buffer_size=10*1024*1024) + unpacker = Unpacker(file_like) for o in unpacker: process(o) Example of streaming deserialize from socket:: - unpacker = Unpacker(max_buffer_size=10*1024*1024) + unpacker = Unpacker(max_buffer_size) while True: buf = sock.recv(1024**2) if not buf: @@ -252,7 +253,7 @@ class Unpacker(object): object_pairs_hook=None, list_hook=None, unicode_errors=None, - max_buffer_size=0, + max_buffer_size=100 * 1024 * 1024, ext_hook=ExtType, max_str_len=-1, max_bin_len=-1, @@ -285,18 +286,20 @@ class Unpacker(object): # state, which _buf_checkpoint records. self._buf_checkpoint = 0 + if not max_buffer_size: + max_buffer_size = 2 ** 31 - 1 if max_str_len == -1: - max_str_len = max_buffer_size or 1024 * 1024 + max_str_len = max_buffer_size if max_bin_len == -1: - max_bin_len = max_buffer_size or 1024 * 1024 + max_bin_len = max_buffer_size if max_array_len == -1: - max_array_len = max_buffer_size or 128 * 1024 + max_array_len = max_buffer_size if max_map_len == -1: - max_map_len = max_buffer_size // 2 or 32 * 1024 + max_map_len = max_buffer_size // 2 if max_ext_len == -1: - max_ext_len = max_buffer_size or 1024 * 1024 + max_ext_len = max_buffer_size - self._max_buffer_size = max_buffer_size or 2 ** 31 - 1 + self._max_buffer_size = max_buffer_size if read_size > self._max_buffer_size: raise ValueError("read_size must be smaller than max_buffer_size") self._read_size = read_size or min(self._max_buffer_size, 16 * 1024) -- cgit v1.2.1