diff options
| -rw-r--r-- | CHANGES | 2 | ||||
| -rw-r--r-- | redis/connection.py | 26 | ||||
| -rw-r--r-- | tests/server_commands.py | 13 |
3 files changed, 38 insertions, 3 deletions
@@ -1,4 +1,6 @@ * 2.4.10 (in development) + * Buffer reads from socket in the PythonParser. Fix for a Windows-specific + bug (#205). * Added the DEBUG OBJECT command. * Added __del__ methods for classes that hold on to resources that need to be cleaned up. This should prevent resource leakage when these objects diff --git a/redis/connection.py b/redis/connection.py index 5a82221..fc95af6 100644 --- a/redis/connection.py +++ b/redis/connection.py @@ -1,10 +1,17 @@ -import errno import socket from itertools import chain, imap from redis.exceptions import ConnectionError, ResponseError, InvalidResponse +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO + + class PythonParser(object): "Plain Python parsing class" + MAX_READ_LENGTH = 1000000 + def __init__(self): self._fp = None @@ -31,7 +38,22 @@ class PythonParser(object): """ try: if length is not None: - return self._fp.read(length+2)[:-2] + bytes_left = length + 2 # read the line ending + if length > self.MAX_READ_LENGTH: + # apparently reading more than 1MB or so from a windows + # socket can cause MemoryErrors. See: + # https://github.com/andymccurdy/redis-py/issues/205 + # read smaller chunks at a time to work around this + buf = StringIO() + while bytes_left > 0: + read_len = min(bytes_left, self.MAX_READ_LENGTH) + buf.write(self._fp.read(read_len)) + bytes_left -= read_len + buf.seek(0) + return buf.read(length) + return self._fp.read(bytes_left)[:-2] + + # no length, read a full line return self._fp.readline()[:-2] except (socket.error, socket.timeout), e: raise ConnectionError("Error while reading from socket: %s" % \ diff --git a/tests/server_commands.py b/tests/server_commands.py index 6cfabe7..dae1fb5 100644 --- a/tests/server_commands.py +++ b/tests/server_commands.py @@ -2,6 +2,7 @@ import redis import unittest import datetime import time +from string import letters from distutils.version import StrictVersion from redis.client import parse_info @@ -99,7 +100,7 @@ class ServerCommandsTestCase(unittest.TestCase): self.assert_(debug_info['serializedlength'] > 0) self.client.rpush('b', 'a1') debug_info = self.client.debug_object('a') - + def test_lastsave(self): self.assert_(isinstance(self.client.lastsave(), datetime.datetime)) @@ -1291,3 +1292,13 @@ class ServerCommandsTestCase(unittest.TestCase): self.assert_('allocation_stats' in parsed) self.assert_('6' in parsed['allocation_stats']) self.assert_('>=256' in parsed['allocation_stats']) + + def test_large_responses(self): + "The PythonParser has some special cases for return values > 1MB" + # load up 5MB of data into a key + data = [] + for i in range(5000000/len(letters)): + data.append(letters) + data = ''.join(data) + self.client.set('a', data) + self.assertEquals(self.client.get('a'), data) |