From a71a24d86aca4747a954ac642a9173a05cedd257 Mon Sep 17 00:00:00 2001 From: Bas Westerbaan Date: Mon, 26 Jan 2015 20:34:31 +0100 Subject: Fix #124 When using Unpacker as an iterator, after each yield, the internal buffer (_fb_buffer) was compacted by reallocation (done by _fb_consume). When dealing with a lot of small objects, this is very ineffecient. Thus in commit 7eb371f8278941fb2323e0c2333ed89c88ab822b the pure python fallback only reallocated the complete buffer when the iteration stops. When halfway there happens to be data missing in the buffer, we rollback the buffer to the state before this failed call, and raise an OutOfData. This rollback, done by _fb_rollback, did not consider the possibility that the buffer was *not* reallocated. This commit corrects that. --- msgpack/fallback.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index d1f39d1..bd99a15 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -195,6 +195,9 @@ class Unpacker(object): # the buffer is not "consumed" completely, for efficiency sake. # Instead, it is done sloppily. To make sure we raise BufferFull at # the correct moments, we have to keep track of how sloppy we were. + # Furthermore, when the buffer is incomplete (that is: in the case + # we raise an OutOfData) we need to rollback the buffer to the correct + # state, which _fb_slopiness records. self._fb_sloppiness = 0 self._max_buffer_size = max_buffer_size or 2**31-1 if read_size > self._max_buffer_size: @@ -283,7 +286,7 @@ class Unpacker(object): def _fb_rollback(self): self._fb_buf_i = 0 - self._fb_buf_o = 0 + self._fb_buf_o = self._fb_sloppiness def _fb_get_extradata(self): bufs = self._fb_buffers[self._fb_buf_i:] -- cgit v1.2.1