Merge pull request #1055 from tzickel/pipeperf

Improve performence of transactions / pipeline requests which involve large chunks of data.

1 files changed, 15 insertions, 8 deletions

diff --git a/redis/connection.py b/redis/connection.py
index 87beb01..2203a9f 100755
--- a/redis/connection.py
+++ b/redis/connection.py
@@ -468,6 +468,7 @@ class Connection(object):
             'db': self.db,
         }
         self._connect_callbacks = []
+        self._buffer_cutoff = 6000
 
     def __repr__(self):
         return self.description_format % self._description_args
@@ -654,10 +655,11 @@ class Connection(object):
 
         buff = SYM_EMPTY.join((SYM_STAR, str(len(args)).encode(), SYM_CRLF))
 
+        buffer_cutoff = self._buffer_cutoff
         for arg in imap(self.encoder.encode, args):
             # to avoid large string mallocs, chunk the command into the
             # output list if we're sending large values
-            if len(buff) > 6000 or len(arg) > 6000:
+            if len(buff) > buffer_cutoff or len(arg) > buffer_cutoff:
                 buff = SYM_EMPTY.join(
                     (buff, SYM_DOLLAR, str(len(arg)).encode(), SYM_CRLF))
                 output.append(buff)
@@ -675,16 +677,21 @@ class Connection(object):
         output = []
         pieces = []
         buffer_length = 0
+        buffer_cutoff = self._buffer_cutoff
 
         for cmd in commands:
             for chunk in self.pack_command(*cmd):
-                pieces.append(chunk)
-                buffer_length += len(chunk)
-
-            if buffer_length > 6000:
-                output.append(SYM_EMPTY.join(pieces))
-                buffer_length = 0
-                pieces = []
+                chunklen = len(chunk)
+                if buffer_length > buffer_cutoff or chunklen > buffer_cutoff:
+                    output.append(SYM_EMPTY.join(pieces))
+                    buffer_length = 0
+                    pieces = []
+
+                if chunklen > self._buffer_cutoff:
+                    output.append(chunk)
+                else:
+                    pieces.append(chunk)
+                    buffer_length += chunklen
 
         if pieces:
             output.append(SYM_EMPTY.join(pieces))