From d91a0d3d68bef38be667ba8e931e3f53cf90be6f Mon Sep 17 00:00:00 2001 From: Bas Westerbaan Date: Tue, 29 Jan 2013 03:45:17 +0100 Subject: Revert "fallback: Use mmap objects instead of strings to unpack" See next commit. This reverts commit 770fed6b7f1ce1f6e9c20881897a417680c8b79a. --- msgpack/fallback.py | 80 +++++++++++++++++++---------------------------------- 1 file changed, 29 insertions(+), 51 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index c0fcc18..2f83a20 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -1,27 +1,15 @@ # Fallback pure Python implementation of msgpack -# -# Easy imports -# import sys import array import struct -# -# Tricky imports -# -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO - -# We will use wStringIO for buffering the writes for packing. -# Normally, we will use cStringIO.StringIO. -# On PyPy we will use PyPy's own StringBuilder. if hasattr(sys, 'pypy_version_info'): + # cStringIO is slow on PyPy, StringIO is faster. However: PyPy's own + # StringBuilder is fastest. from __pypy__.builders import StringBuilder USING_STRINGBUILDER = True - class wStringIO(object): + class StringIO(object): def __init__(self, s=''): if s: self.builder = StringBuilder(len(s)) @@ -34,18 +22,10 @@ if hasattr(sys, 'pypy_version_info'): return self.builder.build() else: USING_STRINGBUILDER = False - wStringIO = StringIO - -# We will use rStringIO for unpacking. -# Normally, this is a mmap. A normal StringIO is not a drop-in replacement --- -# it misses the __len__ operation. -# TODO add fallback for when mmap is unavailable -import mmap -def rStringIO(s): - m = mmap.mmap(-1, len(s)) - m.write(s) - m.seek(0) - return m + try: + from cStringIO import StringIO + except ImportError: + from StringIO import StringIO from msgpack.exceptions import ( BufferFull, @@ -204,13 +184,13 @@ class Unpacker(object): if self._fb_buf_n + len(next_bytes) > self.max_buffer_size: raise BufferFull self._fb_buf_n += len(next_bytes) - self._fb_buffers.append(rStringIO(next_bytes)) + self._fb_buffers.append(next_bytes) def _fb_consume(self): self._fb_buffers = self._fb_buffers[self._fb_buf_i:] if self._fb_buffers: - self._fb_buffers[0] = rStringIO(self._fb_buffers[0][ - self._fb_buffers[0].tell():]) + self._fb_buffers[0] = self._fb_buffers[0][self._fb_buf_o:] + self._fb_buf_o = 0 self._fb_buf_i = 0 self._fb_buf_n = sum(map(len, self._fb_buffers)) @@ -232,20 +212,16 @@ class Unpacker(object): return self._fb_read(n) def _fb_rollback(self): - for buf in self._fb_buffers: - buf.seek(0) self._fb_buf_i = 0 + self._fb_buf_o = 0 def _fb_get_extradata(self): bufs = self._fb_buffers[self._fb_buf_i:] if bufs: - bufs[0] = rStringIO(bufs[0][bufs[0].tell():]) - return ''.join([buf[:] for buf in bufs]) + bufs[0] = bufs[0][self._fb_buf_o:] + return ''.join(bufs) def _fb_read(self, n, write_bytes=None): - if (write_bytes is None and self._fb_buf_i < len(self._fb_buffers) - and self._fb_buffers[0].tell() + n < len(self._fb_buffers[0])): - return self._fb_buffers[0].read(n) ret = '' while len(ret) != n: if self._fb_buf_i == len(self._fb_buffers): @@ -254,12 +230,14 @@ class Unpacker(object): tmp = self.file_like.read(self.read_size) if not tmp: break - self._fb_buffers.append(rStringIO(tmp)) + self._fb_buffers.append(tmp) continue sliced = n - len(ret) - ret += self._fb_buffers[self._fb_buf_i].read(sliced) - if (self._fb_buffers[self._fb_buf_i].tell() - == len(self._fb_buffers[self._fb_buf_i])): + ret += self._fb_buffers[self._fb_buf_i][ + self._fb_buf_o:self._fb_buf_o + sliced] + self._fb_buf_o += sliced + if self._fb_buf_o >= len(self._fb_buffers[self._fb_buf_i]): + self._fb_buf_o = 0 self._fb_buf_i += 1 if len(ret) != n: self._fb_rollback() @@ -416,7 +394,7 @@ class Packer(object): self.autoreset = autoreset self.encoding = encoding self.unicode_errors = unicode_errors - self.buffer = wStringIO() + self.buffer = StringIO() if default is not None: if not callable(default): raise TypeError("default must be callable") @@ -486,33 +464,33 @@ class Packer(object): self._pack(obj) ret = self.buffer.getvalue() if self.autoreset: - self.buffer = wStringIO() + self.buffer = StringIO() elif USING_STRINGBUILDER: - self.buffer = wStringIO(ret) + self.buffer = StringIO(ret) return ret def pack_map_pairs(self, pairs): self._fb_pack_map_pairs(len(pairs), pairs) ret = self.buffer.getvalue() if self.autoreset: - self.buffer = wStringIO() + self.buffer = StringIO() elif USING_STRINGBUILDER: - self.buffer = wStringIO(ret) + self.buffer = StringIO(ret) return ret def pack_array_header(self, n): self._fb_pack_array_header(n) ret = self.buffer.getvalue() if self.autoreset: - self.buffer = wStringIO() + self.buffer = StringIO() elif USING_STRINGBUILDER: - self.buffer = wStringIO(ret) + self.buffer = StringIO(ret) return ret def pack_map_header(self, n): self._fb_pack_map_header(n) ret = self.buffer.getvalue() if self.autoreset: - self.buffer = wStringIO() + self.buffer = StringIO() elif USING_STRINGBUILDER: - self.buffer = wStringIO(ret) + self.buffer = StringIO(ret) return ret def _fb_pack_array_header(self, n): if n <= 0x0f: @@ -538,4 +516,4 @@ class Packer(object): def bytes(self): return self.buffer.getvalue() def reset(self): - self.buffer = wStringIO() + self.buffer = StringIO() -- cgit v1.2.1