From 5c07e3d68036b6fe32687805f1ff83fb3a25128c Mon Sep 17 00:00:00 2001 From: tzickel Date: Sat, 14 Mar 2015 14:49:50 +0200 Subject: Improve performence of transactions / pipeline requests which involve large chunks of data. --- redis/connection.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/redis/connection.py b/redis/connection.py index 00c3311..67b6756 100755 --- a/redis/connection.py +++ b/redis/connection.py @@ -461,6 +461,7 @@ class Connection(object): 'db': self.db, } self._connect_callbacks = [] + self._buffer_cutoff = 6000 def __repr__(self): return self.description_format % self._description_args @@ -648,10 +649,11 @@ class Connection(object): buff = SYM_EMPTY.join( (SYM_STAR, b(str(len(args))), SYM_CRLF)) + buffer_cutoff = self._buffer_cutoff for arg in imap(self.encoder.encode, args): # to avoid large string mallocs, chunk the command into the # output list if we're sending large values - if len(buff) > 6000 or len(arg) > 6000: + if len(buff) > buffer_cutoff or len(arg) > buffer_cutoff: buff = SYM_EMPTY.join( (buff, SYM_DOLLAR, b(str(len(arg))), SYM_CRLF)) output.append(buff) @@ -668,16 +670,21 @@ class Connection(object): output = [] pieces = [] buffer_length = 0 + buffer_cutoff = self._buffer_cutoff for cmd in commands: for chunk in self.pack_command(*cmd): - pieces.append(chunk) - buffer_length += len(chunk) - - if buffer_length > 6000: - output.append(SYM_EMPTY.join(pieces)) - buffer_length = 0 - pieces = [] + chunklen = len(chunk) + if buffer_length > buffer_cutoff or chunklen > buffer_cutoff: + output.append(SYM_EMPTY.join(pieces)) + buffer_length = 0 + pieces = [] + + if chunklen > self._buffer_cutoff: + output.append(chunk) + else: + pieces.append(chunk) + buffer_length += chunklen if pieces: output.append(SYM_EMPTY.join(pieces)) -- cgit v1.2.1