summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSean Reifschneider <jafo@tummy.com>2014-03-25 22:45:16 -0600
committerSean Reifschneider <jafo@tummy.com>2014-03-25 22:45:16 -0600
commit179eeca44bfae70cfc4d7f7d2fd10d808f47416f (patch)
tree8c98b8bfaa3e0575abf7bbb195b2c9f7699f2be7
parente3468928c53588e66bfabcbe6fde639222de02b3 (diff)
parent853040beb9d2b709e67cae77fffd08dceab8b61a (diff)
downloadpython-memcached-179eeca44bfae70cfc4d7f7d2fd10d808f47416f.tar.gz
Merge branch 'py33_fixes' of https://github.com/cabrera/python-memcached into cabrera-py33_fixes
Conflicts: README.md memcache.py tests/test_setmulti.py
-rw-r--r--.gitignore2
-rw-r--r--README.md13
-rw-r--r--memcache.py950
-rw-r--r--requirements.txt1
-rw-r--r--setup.py24
-rw-r--r--test-requirements.txt3
-rw-r--r--tests/Makefile7
-rw-r--r--tests/test_setmulti.py12
-rw-r--r--tox.ini24
9 files changed, 578 insertions, 458 deletions
diff --git a/.gitignore b/.gitignore
index e18136f..d705d4e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,5 @@
*.pyc
/dist
/python_memcached.egg-info
+.tox
+.coverage
diff --git a/README.md b/README.md
index 0811f8a..115115e 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,8 @@
[![Build
Status](https://travis-ci.org/linsomniac/python-memcached.svg)](https://travis-ci.org/linsomniac/python-memcached)
+## Overview
+
This software is a 100% Python interface to the memcached memory cache
daemon. It is the client side software which allows storing values
in one or more, possibly remote, memcached servers. Search google for
@@ -17,3 +19,14 @@ Please report issues and submit code changes to the github repository at:
For changes prior to 2013-03-26, see the old Launchpad repository at:
Historic issues: https://launchpad.net/python-memcached
+
+## Testing
+
+Test patches locally and easily by running tox:
+
+ pip install tox
+ tox -e py27
+
+Test for style by running tox:
+
+ tox -e pep8
diff --git a/memcache.py b/memcache.py
index 769610e..ea96b40 100644
--- a/memcache.py
+++ b/memcache.py
@@ -1,12 +1,12 @@
#!/usr/bin/env python
-"""
-client module for memcached (memory cache daemon)
+"""client module for memcached (memory cache daemon)
Overview
========
-See U{the MemCached homepage<http://www.danga.com/memcached>} for more about memcached.
+See U{the MemCached homepage<http://www.danga.com/memcached>} for more
+about memcached.
Usage summary
=============
@@ -22,7 +22,8 @@ This should give you a feel for how this module operates::
mc.set("another_key", 3)
mc.delete("another_key")
- mc.set("key", "1") # note that the key used for incr/decr must be a string.
+ mc.set("key", "1") # note that the key used for incr/decr must be
+ # a string.
mc.incr("key")
mc.decr("key")
@@ -41,57 +42,50 @@ Detailed Documentation
======================
More detailed documentation is available in the L{Client} class.
+
"""
-import sys
-import socket
-import time
+from __future__ import print_function
+import binascii
+import io
import os
+import pickle
import re
-try:
- import cPickle as pickle
-except ImportError:
- import pickle
+import socket
+import sys
+import threading
+import time
+import zlib
+
+import six
+
-from binascii import crc32 # zlib version is not cross-platform
def cmemcache_hash(key):
- return((((crc32(key) & 0xffffffff) >> 16) & 0x7fff) or 1)
+ return ((((binascii.crc32(key.encode('ascii')) & 0xffffffff) >> 16) & 0x7fff) or 1)
serverHashFunction = cmemcache_hash
+
def useOldServerHashFunction():
"""Use the old python-memcache server hash function."""
global serverHashFunction
- serverHashFunction = crc32
-
-try:
- from zlib import compress, decompress
- _supports_compress = True
-except ImportError:
- _supports_compress = False
- # quickly define a decompress just in case we recv compressed data.
- def decompress(val):
- raise _Error("received compressed data but I don't support compression (import error)")
-
-try:
- from cStringIO import StringIO
-except ImportError:
- from StringIO import StringIO
+ serverHashFunction = binascii.crc32
+
valid_key_chars_re = re.compile('[\x21-\x7e\x80-\xff]+$')
# Original author: Evan Martin of Danga Interactive
-__author__ = "Sean Reifschneider <jafo-memcached@tummy.com>"
+__author__ = "Sean Reifschneider <jafo-memcached@tummy.com>"
__version__ = "1.53"
__copyright__ = "Copyright (C) 2003 Danga Interactive"
# http://en.wikipedia.org/wiki/Python_Software_Foundation_License
-__license__ = "Python Software Foundation License"
+__license__ = "Python Software Foundation License"
SERVER_MAX_KEY_LENGTH = 250
-# Storing values larger than 1MB requires recompiling memcached. If you do,
-# this value can be changed by doing "memcache.SERVER_MAX_VALUE_LENGTH = N"
-# after importing this module.
-SERVER_MAX_VALUE_LENGTH = 1024*1024
+# Storing values larger than 1MB requires recompiling memcached. If
+# you do, this value can be changed by doing
+# "memcache.SERVER_MAX_VALUE_LENGTH = N" after importing this module.
+SERVER_MAX_VALUE_LENGTH = 1024 * 1024
class _Error(Exception):
@@ -102,58 +96,57 @@ class _ConnectionDeadError(Exception):
pass
-try:
- # Only exists in Python 2.4+
- from threading import local
-except ImportError:
- # TODO: add the pure-python local implementation
- class local(object):
- pass
-
-
_DEAD_RETRY = 30 # number of seconds before retrying a dead server.
-_SOCKET_TIMEOUT = 3 # number of seconds before sockets timeout.
+_SOCKET_TIMEOUT = 3 # number of seconds before sockets timeout.
-class Client(local):
- """
- Object representing a pool of memcache servers.
+class Client(threading.local):
+ """Object representing a pool of memcache servers.
See L{memcache} for an overview.
In all cases where a key is used, the key can be either:
1. A simple hashable type (string, integer, etc.).
- 2. A tuple of C{(hashvalue, key)}. This is useful if you want to avoid
- making this module calculate a hash value. You may prefer, for
- example, to keep all of a given user's objects on the same memcache
- server, so you could use the user's unique id as the hash value.
+ 2. A tuple of C{(hashvalue, key)}. This is useful if you want
+ to avoid making this module calculate a hash value. You may
+ prefer, for example, to keep all of a given user's objects on
+ the same memcache server, so you could use the user's unique
+ id as the hash value.
+
- @group Setup: __init__, set_servers, forget_dead_hosts, disconnect_all, debuglog
+ @group Setup: __init__, set_servers, forget_dead_hosts,
+ disconnect_all, debuglog
@group Insertion: set, add, replace, set_multi
@group Retrieval: get, get_multi
@group Integers: incr, decr
@group Removal: delete, delete_multi
- @sort: __init__, set_servers, forget_dead_hosts, disconnect_all, debuglog,\
- set, set_multi, add, replace, get, get_multi, incr, decr, delete, delete_multi
+ @sort: __init__, set_servers, forget_dead_hosts, disconnect_all,
+ debuglog,\ set, set_multi, add, replace, get, get_multi,
+ incr, decr, delete, delete_multi
"""
- _FLAG_PICKLE = 1<<0
- _FLAG_INTEGER = 1<<1
- _FLAG_LONG = 1<<2
- _FLAG_COMPRESSED = 1<<3
+ _FLAG_PICKLE = 1 << 0
+ _FLAG_INTEGER = 1 << 1
+ _FLAG_LONG = 1 << 2
+ _FLAG_COMPRESSED = 1 << 3
_SERVER_RETRIES = 10 # how many times to try finding a free server.
# exceptions for Client
class MemcachedKeyError(Exception):
pass
+
class MemcachedKeyLengthError(MemcachedKeyError):
pass
+
class MemcachedKeyCharacterError(MemcachedKeyError):
pass
+
class MemcachedKeyNoneError(MemcachedKeyError):
pass
+
class MemcachedKeyTypeError(MemcachedKeyError):
pass
+
class MemcachedStringEncodingError(Exception):
pass
@@ -162,43 +155,51 @@ class Client(local):
pload=None, pid=None,
server_max_key_length=None, server_max_value_length=None,
dead_retry=_DEAD_RETRY, socket_timeout=_SOCKET_TIMEOUT,
- cache_cas = False, flush_on_reconnect=0, check_keys=True):
- """
- Create a new Client object with the given list of servers.
+ cache_cas=False, flush_on_reconnect=0, check_keys=True):
+ """Create a new Client object with the given list of servers.
@param servers: C{servers} is passed to L{set_servers}.
- @param debug: whether to display error messages when a server can't be
- contacted.
- @param pickleProtocol: number to mandate protocol used by (c)Pickle.
- @param pickler: optional override of default Pickler to allow subclassing.
- @param unpickler: optional override of default Unpickler to allow subclassing.
- @param pload: optional persistent_load function to call on pickle loading.
- Useful for cPickle since subclassing isn't allowed.
- @param pid: optional persistent_id function to call on pickle storing.
- Useful for cPickle since subclassing isn't allowed.
- @param dead_retry: number of seconds before retrying a blacklisted
- server. Default to 30 s.
- @param socket_timeout: timeout in seconds for all calls to a server. Defaults
- to 3 seconds.
- @param cache_cas: (default False) If true, cas operations will be
- cached. WARNING: This cache is not expired internally, if you have
- a long-running process you will need to expire it manually via
- client.reset_cas(), or the cache can grow unlimited.
+ @param debug: whether to display error messages when a server
+ can't be contacted.
+ @param pickleProtocol: number to mandate protocol used by
+ (c)Pickle.
+ @param pickler: optional override of default Pickler to allow
+ subclassing.
+ @param unpickler: optional override of default Unpickler to
+ allow subclassing.
+ @param pload: optional persistent_load function to call on
+ pickle loading. Useful for cPickle since subclassing isn't
+ allowed.
+ @param pid: optional persistent_id function to call on pickle
+ storing. Useful for cPickle since subclassing isn't allowed.
+ @param dead_retry: number of seconds before retrying a
+ blacklisted server. Default to 30 s.
+ @param socket_timeout: timeout in seconds for all calls to a
+ server. Defaults to 3 seconds.
+ @param cache_cas: (default False) If true, cas operations will
+ be cached. WARNING: This cache is not expired internally, if
+ you have a long-running process you will need to expire it
+ manually via client.reset_cas(), or the cache can grow
+ unlimited.
@param server_max_key_length: (default SERVER_MAX_KEY_LENGTH)
Data that is larger than this will not be sent to the server.
- @param server_max_value_length: (default SERVER_MAX_VALUE_LENGTH)
- Data that is larger than this will not be sent to the server.
- @param flush_on_reconnect: optional flag which prevents a scenario that
- can cause stale data to be read: If there's more than one memcached
- server and the connection to one is interrupted, keys that mapped to
- that server will get reassigned to another. If the first server comes
- back, those keys will map to it again. If it still has its data, get()s
- can read stale data that was overwritten on another server. This flag
- is off by default for backwards compatibility.
- @param check_keys: (default True) If True, the key is checked to
- ensure it is the correct length and composed of the right characters.
+ @param server_max_value_length: (default
+ SERVER_MAX_VALUE_LENGTH) Data that is larger than this will
+ not be sent to the server.
+ @param flush_on_reconnect: optional flag which prevents a
+ scenario that can cause stale data to be read: If there's more
+ than one memcached server and the connection to one is
+ interrupted, keys that mapped to that server will get
+ reassigned to another. If the first server comes back, those
+ keys will map to it again. If it still has its data, get()s
+ can read stale data that was overwritten on another
+ server. This flag is off by default for backwards
+ compatibility.
+ @param check_keys: (default True) If True, the key is checked
+ to ensure it is the correct length and composed of the right
+ characters.
"""
- local.__init__(self)
+ super(Client, self).__init__()
self.debug = debug
self.dead_retry = dead_retry
self.socket_timeout = socket_timeout
@@ -223,69 +224,73 @@ class Client(local):
self.server_max_value_length = SERVER_MAX_VALUE_LENGTH
# figure out the pickler style
- file = StringIO()
+ file = io.StringIO()
try:
- pickler = self.pickler(file, protocol = self.pickleProtocol)
+ pickler = self.pickler(file, protocol=self.pickleProtocol)
self.picklerIsKeyword = True
except TypeError:
self.picklerIsKeyword = False
def reset_cas(self):
- """
- Reset the cas cache. This is only used if the Client() object
- was created with "cache_cas=True". If used, this cache does not
- expire internally, so it can grow unbounded if you do not clear it
+ """Reset the cas cache.
+
+ This is only used if the Client() object was created with
+ "cache_cas=True". If used, this cache does not expire
+ internally, so it can grow unbounded if you do not clear it
yourself.
"""
self.cas_ids = {}
-
def set_servers(self, servers):
- """
- Set the pool of servers used by this client.
+ """Set the pool of servers used by this client.
@param servers: an array of servers.
Servers can be passed in two forms:
- 1. Strings of the form C{"host:port"}, which implies a default weight of 1.
- 2. Tuples of the form C{("host:port", weight)}, where C{weight} is
- an integer weight value.
+ 1. Strings of the form C{"host:port"}, which implies a
+ default weight of 1.
+ 2. Tuples of the form C{("host:port", weight)}, where
+ C{weight} is an integer weight value.
+
"""
self.servers = [_Host(s, self.debug, dead_retry=self.dead_retry,
- socket_timeout=self.socket_timeout,
- flush_on_reconnect=self.flush_on_reconnect)
- for s in servers]
+ socket_timeout=self.socket_timeout,
+ flush_on_reconnect=self.flush_on_reconnect)
+ for s in servers]
self._init_buckets()
- def get_stats(self, stat_args = None):
- '''Get statistics from each of the servers.
+ def get_stats(self, stat_args=None):
+ """Get statistics from each of the servers.
@param stat_args: Additional arguments to pass to the memcache
"stats" command.
- @return: A list of tuples ( server_identifier, stats_dictionary ).
- The dictionary contains a number of name/value pairs specifying
- the name of the status field and the string value associated with
- it. The values are not converted from strings.
- '''
+ @return: A list of tuples ( server_identifier,
+ stats_dictionary ). The dictionary contains a number of
+ name/value pairs specifying the name of the status field
+ and the string value associated with it. The values are
+ not converted from strings.
+ """
data = []
for s in self.servers:
- if not s.connect(): continue
+ if not s.connect():
+ continue
if s.family == socket.AF_INET:
- name = '%s:%s (%s)' % ( s.ip, s.port, s.weight )
+ name = '%s:%s (%s)' % (s.ip, s.port, s.weight)
elif s.family == socket.AF_INET6:
- name = '[%s]:%s (%s)' % ( s.ip, s.port, s.weight )
+ name = '[%s]:%s (%s)' % (s.ip, s.port, s.weight)
else:
- name = 'unix:%s (%s)' % ( s.address, s.weight )
+ name = 'unix:%s (%s)' % (s.address, s.weight)
if not stat_args:
s.send_cmd('stats')
else:
s.send_cmd('stats ' + stat_args)
serverData = {}
- data.append(( name, serverData ))
+ data.append((name, serverData))
readline = s.readline
while 1:
line = readline()
- if not line or line.strip() == 'END': break
+ if not line or line.strip() == 'END':
+ break
stats = line.split(' ', 2)
serverData[stats[1]] = stats[2]
@@ -294,24 +299,26 @@ class Client(local):
def get_slabs(self):
data = []
for s in self.servers:
- if not s.connect(): continue
+ if not s.connect():
+ continue
if s.family == socket.AF_INET:
- name = '%s:%s (%s)' % ( s.ip, s.port, s.weight )
+ name = '%s:%s (%s)' % (s.ip, s.port, s.weight)
elif s.family == socket.AF_INET6:
- name = '[%s]:%s (%s)' % ( s.ip, s.port, s.weight )
+ name = '[%s]:%s (%s)' % (s.ip, s.port, s.weight)
else:
- name = 'unix:%s (%s)' % ( s.address, s.weight )
+ name = 'unix:%s (%s)' % (s.address, s.weight)
serverData = {}
- data.append(( name, serverData ))
+ data.append((name, serverData))
s.send_cmd('stats items')
readline = s.readline
while 1:
line = readline()
- if not line or line.strip() == 'END': break
+ if not line or line.strip() == 'END':
+ break
item = line.split(' ', 2)
- #0 = STAT, 1 = ITEM, 2 = Value
+ # 0 = STAT, 1 = ITEM, 2 = Value
slab = item[1].split(':', 2)
- #0 = items, 1 = Slab #, 2 = Name
+ # 0 = items, 1 = Slab #, 2 = Name
if slab[1] not in serverData:
serverData[slab[1]] = {}
serverData[slab[1]][slab[2]] = item[2]
@@ -320,7 +327,8 @@ class Client(local):
def flush_all(self):
"""Expire all data in memcache servers that are reachable."""
for s in self.servers:
- if not s.connect(): continue
+ if not s.connect():
+ continue
s.flush()
def debuglog(self, str):
@@ -334,9 +342,7 @@ class Client(local):
self.stats[func] += 1
def forget_dead_hosts(self):
- """
- Reset every host in the pool to an "alive" state.
- """
+ """Reset every host in the pool to an "alive" state."""
for s in self.servers:
s.deaduntil = 0
@@ -358,7 +364,7 @@ class Client(local):
for i in range(Client._SERVER_RETRIES):
server = self.buckets[serverhash % len(self.buckets)]
if server.connect():
- #print "(using server %s)" % server,
+ # print("(using server %s)" % server,)
return server, key
serverhash = serverHashFunction(str(serverhash) + str(i))
return None, None
@@ -368,54 +374,54 @@ class Client(local):
s.close_socket()
def delete_multi(self, keys, time=0, key_prefix=''):
- '''
- Delete multiple keys in the memcache doing just one query.
+ """Delete multiple keys in the memcache doing just one query.
- >>> notset_keys = mc.set_multi({'key1' : 'val1', 'key2' : 'val2'})
- >>> mc.get_multi(['key1', 'key2']) == {'key1' : 'val1', 'key2' : 'val2'}
+ >>> notset_keys = mc.set_multi({'a1' : 'val1', 'a2' : 'val2'})
+ >>> mc.get_multi(['a1', 'a2']) == {'a1' : 'val1','a2' : 'val2'}
1
>>> mc.delete_multi(['key1', 'key2'])
1
>>> mc.get_multi(['key1', 'key2']) == {}
1
-
- This method is recommended over iterated regular L{delete}s as it reduces total latency, since
- your app doesn't have to wait for each round-trip of L{delete} before sending
- the next one.
+ This method is recommended over iterated regular L{delete}s as
+ it reduces total latency, since your app doesn't have to wait
+ for each round-trip of L{delete} before sending the next one.
@param keys: An iterable of keys to clear
- @param time: number of seconds any subsequent set / update commands should fail. Defaults to 0 for no delay.
- @param key_prefix: Optional string to prepend to each key when sending to memcache.
- See docs for L{get_multi} and L{set_multi}.
-
+ @param time: number of seconds any subsequent set / update
+ commands should fail. Defaults to 0 for no delay.
+ @param key_prefix: Optional string to prepend to each key when
+ sending to memcache. See docs for L{get_multi} and
+ L{set_multi}.
@return: 1 if no failure in communication with any memcacheds.
@rtype: int
-
- '''
+ """
self._statlog('delete_multi')
- server_keys, prefixed_to_orig_key = self._map_and_prefix_keys(keys, key_prefix)
+ server_keys, prefixed_to_orig_key = self._map_and_prefix_keys(
+ keys, key_prefix)
# send out all requests on each server before reading anything
dead_servers = []
rc = 1
- for server in server_keys.iterkeys():
+ for server in six.iterkeys(server_keys):
bigcmd = []
write = bigcmd.append
- if time != None:
- for key in server_keys[server]: # These are mangled keys
- write("delete %s %d\r\n" % (key, time))
+ if time is not None:
+ for key in server_keys[server]: # These are mangled keys
+ write("delete %s %d\r\n" % (key, time))
else:
- for key in server_keys[server]: # These are mangled keys
- write("delete %s\r\n" % key)
+ for key in server_keys[server]: # These are mangled keys
+ write("delete %s\r\n" % key)
try:
server.send_cmds(''.join(bigcmd))
- except socket.error, msg:
+ except socket.error as msg:
rc = 0
- if isinstance(msg, tuple): msg = msg[1]
+ if isinstance(msg, tuple):
+ msg = msg[1]
server.mark_dead(msg)
dead_servers.append(server)
@@ -427,8 +433,9 @@ class Client(local):
try:
for key in keys:
server.expect("DELETED")
- except socket.error, msg:
- if isinstance(msg, tuple): msg = msg[1]
+ except socket.error as msg:
+ if isinstance(msg, tuple):
+ msg = msg[1]
server.mark_dead(msg)
rc = 0
return rc
@@ -463,7 +470,7 @@ class Client(local):
if not server:
return 0
self._statlog(cmd)
- if time != None and time != 0:
+ if time is not None and time != 0:
cmd = "%s %s %d" % (cmd, key, time)
else:
cmd = "%s %s" % (cmd, key)
@@ -471,23 +478,25 @@ class Client(local):
try:
server.send_cmd(cmd)
line = server.readline()
- if line and line.strip() in expected: return 1
+ if line and line.strip() in expected:
+ return 1
self.debuglog('%s expected %s, got: %s'
% (cmd, ' or '.join(expected), repr(line)))
except socket.error, msg:
- if isinstance(msg, tuple): msg = msg[1]
+ if isinstance(msg, tuple):
+ msg = msg[1]
server.mark_dead(msg)
return 0
def incr(self, key, delta=1):
- """
- Sends a command to the server to atomically increment the value
- for C{key} by C{delta}, or by 1 if C{delta} is unspecified.
- Returns None if C{key} doesn't exist on server, otherwise it
- returns the new value after incrementing.
+ """Sends a command to the server to atomically increment the
+ value for C{key} by C{delta}, or by 1 if C{delta} is
+ unspecified. Returns None if C{key} doesn't exist on server,
+ otherwise it returns the new value after incrementing.
- Note that the value for C{key} must already exist in the memcache,
- and it must be the string representation of an integer.
+ Note that the value for C{key} must already exist in the
+ memcache, and it must be the string representation of an
+ integer.
>>> mc.set("counter", "20") # returns 1, indicating success
1
@@ -496,22 +505,25 @@ class Client(local):
>>> mc.incr("counter")
22
- Overflow on server is not checked. Be aware of values approaching
- 2**32. See L{decr}.
+ Overflow on server is not checked. Be aware of values
+ approaching 2**32. See L{decr}.
+
+ @param delta: Integer amount to increment by (should be zero
+ or greater).
- @param delta: Integer amount to increment by (should be zero or greater).
@return: New value after incrementing.
@rtype: int
"""
return self._incrdecr("incr", key, delta)
def decr(self, key, delta=1):
- """
- Like L{incr}, but decrements. Unlike L{incr}, underflow is checked and
- new values are capped at 0. If server value is 1, a decrement of 2
- returns 0, not -1.
+ """Like L{incr}, but decrements. Unlike L{incr}, underflow is
+ checked and new values are capped at 0. If server value is 1,
+ a decrement of 2 returns 0, not -1.
+
+ @param delta: Integer amount to decrement by (should be zero
+ or greater).
- @param delta: Integer amount to decrement by (should be zero or greater).
@return: New value after decrementing or None on error.
@rtype: int
"""
@@ -528,18 +540,20 @@ class Client(local):
try:
server.send_cmd(cmd)
line = server.readline()
- if line == None or line.strip() =='NOT_FOUND': return None
+ if line is None or line.strip() == 'NOT_FOUND':
+ return None
return int(line)
- except socket.error, msg:
- if isinstance(msg, tuple): msg = msg[1]
+ except socket.error as msg:
+ if isinstance(msg, tuple):
+ msg = msg[1]
server.mark_dead(msg)
return None
- def add(self, key, val, time = 0, min_compress_len = 0):
- '''
- Add new key with value.
+ def add(self, key, val, time=0, min_compress_len=0):
+ '''Add new key with value.
- Like L{set}, but only stores in memcache if the key doesn't already exist.
+ Like L{set}, but only stores in memcache if the key doesn't
+ already exist.
@return: Nonzero on success.
@rtype: int
@@ -583,67 +597,73 @@ class Client(local):
'''Unconditionally sets a key to a given value in the memcache.
The C{key} can optionally be an tuple, with the first element
- being the server hash value and the second being the key.
- If you want to avoid making this module calculate a hash value.
- You may prefer, for example, to keep all of a given user's objects
- on the same memcache server, so you could use the user's unique
- id as the hash value.
+ being the server hash value and the second being the key. If
+ you want to avoid making this module calculate a hash value.
+ You may prefer, for example, to keep all of a given user's
+ objects on the same memcache server, so you could use the
+ user's unique id as the hash value.
@return: Nonzero on success.
@rtype: int
- @param time: Tells memcached the time which this value should expire, either
- as a delta number of seconds, or an absolute unix time-since-the-epoch
- value. See the memcached protocol docs section "Storage Commands"
- for more info on <exptime>. We default to 0 == cache forever.
- @param min_compress_len: The threshold length to kick in auto-compression
- of the value using the zlib.compress() routine. If the value being cached is
- a string, then the length of the string is measured, else if the value is an
- object, then the length of the pickle result is measured. If the resulting
- attempt at compression yeilds a larger string than the input, then it is
- discarded. For backwards compatability, this parameter defaults to 0,
- indicating don't ever try to compress.
+
+ @param time: Tells memcached the time which this value should
+ expire, either as a delta number of seconds, or an absolute
+ unix time-since-the-epoch value. See the memcached protocol
+ docs section "Storage Commands" for more info on <exptime>. We
+ default to 0 == cache forever.
+
+ @param min_compress_len: The threshold length to kick in
+ auto-compression of the value using the zlib.compress()
+ routine. If the value being cached is a string, then the
+ length of the string is measured, else if the value is an
+ object, then the length of the pickle result is measured. If
+ the resulting attempt at compression yeilds a larger string
+ than the input, then it is discarded. For backwards
+ compatability, this parameter defaults to 0, indicating don't
+ ever try to compress.
+
'''
return self._set("set", key, val, time, min_compress_len)
-
def cas(self, key, val, time=0, min_compress_len=0):
'''Sets a key to a given value in the memcache if it hasn't been
altered since last fetched. (See L{gets}).
The C{key} can optionally be an tuple, with the first element
- being the server hash value and the second being the key.
- If you want to avoid making this module calculate a hash value.
- You may prefer, for example, to keep all of a given user's objects
- on the same memcache server, so you could use the user's unique
- id as the hash value.
+ being the server hash value and the second being the key. If
+ you want to avoid making this module calculate a hash value.
+ You may prefer, for example, to keep all of a given user's
+ objects on the same memcache server, so you could use the
+ user's unique id as the hash value.
@return: Nonzero on success.
@rtype: int
- @param time: Tells memcached the time which this value should expire,
- either as a delta number of seconds, or an absolute unix
- time-since-the-epoch value. See the memcached protocol docs section
- "Storage Commands" for more info on <exptime>. We default to
- 0 == cache forever.
+
+ @param time: Tells memcached the time which this value should
+ expire, either as a delta number of seconds, or an absolute
+ unix time-since-the-epoch value. See the memcached protocol
+ docs section "Storage Commands" for more info on <exptime>. We
+ default to 0 == cache forever.
+
@param min_compress_len: The threshold length to kick in
- auto-compression of the value using the zlib.compress() routine. If
- the value being cached is a string, then the length of the string is
- measured, else if the value is an object, then the length of the
- pickle result is measured. If the resulting attempt at compression
- yeilds a larger string than the input, then it is discarded. For
- backwards compatability, this parameter defaults to 0, indicating
- don't ever try to compress.
+ auto-compression of the value using the zlib.compress()
+ routine. If the value being cached is a string, then the
+ length of the string is measured, else if the value is an
+ object, then the length of the pickle result is measured. If
+ the resulting attempt at compression yeilds a larger string
+ than the input, then it is discarded. For backwards
+ compatability, this parameter defaults to 0, indicating don't
+ ever try to compress.
'''
return self._set("cas", key, val, time, min_compress_len)
-
def _map_and_prefix_keys(self, key_iterable, key_prefix):
- """Compute the mapping of server (_Host instance) -> list of keys to stuff onto that server, as well as the mapping of
- prefixed key -> original key.
-
-
+ """Compute the mapping of server (_Host instance) -> list of keys to
+ stuff onto that server, as well as the mapping of prefixed key
+ -> original key.
"""
# Check it just once ...
- key_extra_len=len(key_prefix)
+ key_extra_len = len(key_prefix)
if key_prefix and self.do_check_key:
self.check_key(key_prefix)
@@ -654,12 +674,18 @@ class Client(local):
# build up a list for each server of all the keys we want.
for orig_key in key_iterable:
if isinstance(orig_key, tuple):
- # Tuple of hashvalue, key ala _get_server(). Caller is essentially telling us what server to stuff this on.
+ # Tuple of hashvalue, key ala _get_server(). Caller is
+ # essentially telling us what server to stuff this on.
# Ensure call to _get_server gets a Tuple as well.
str_orig_key = str(orig_key[1])
- server, key = self._get_server((orig_key[0], key_prefix + str_orig_key)) # Gotta pre-mangle key before hashing to a server. Returns the mangled key.
+
+ # Gotta pre-mangle key before hashing to a
+ # server. Returns the mangled key.
+ server, key = self._get_server(
+ (orig_key[0], key_prefix + str_orig_key))
else:
- str_orig_key = str(orig_key) # set_multi supports int / long keys.
+ # set_multi supports int / long keys.
+ str_orig_key = str(orig_key)
server, key = self._get_server(key_prefix + str_orig_key)
# Now check to make sure key length is proper ...
@@ -677,34 +703,39 @@ class Client(local):
return (server_keys, prefixed_to_orig_key)
def set_multi(self, mapping, time=0, key_prefix='', min_compress_len=0):
- '''
- Sets multiple keys in the memcache doing just one query.
+ '''Sets multiple keys in the memcache doing just one query.
>>> notset_keys = mc.set_multi({'key1' : 'val1', 'key2' : 'val2'})
- >>> mc.get_multi(['key1', 'key2']) == {'key1' : 'val1', 'key2' : 'val2'}
+ >>> mc.get_multi(['key1', 'key2']) == {'key1' : 'val1',
+ ... 'key2' : 'val2'}
1
- This method is recommended over regular L{set} as it lowers the
- number of total packets flying around your network, reducing
- total latency, since your app doesn't have to wait for each
- round-trip of L{set} before sending the next one.
+ This method is recommended over regular L{set} as it lowers
+ the number of total packets flying around your network,
+ reducing total latency, since your app doesn't have to wait
+ for each round-trip of L{set} before sending the next one.
@param mapping: A dict of key/value pairs to set.
+
@param time: Tells memcached the time which this value should
- expire, either as a delta number of seconds, or an absolute
- unix time-since-the-epoch value. See the memcached protocol
- docs section "Storage Commands" for more info on <exptime>. We
- default to 0 == cache forever.
- @param key_prefix: Optional string to prepend to each key when
+ expire, either as a delta number of seconds, or an
+ absolute unix time-since-the-epoch value. See the
+ memcached protocol docs section "Storage Commands" for
+ more info on <exptime>. We default to 0 == cache forever.
+
+ @param key_prefix: Optional string to prepend to each key when
sending to memcache. Allows you to efficiently stuff these
keys into a pseudo-namespace in memcache:
>>> notset_keys = mc.set_multi(
- ... {'key1' : 'val1', 'key2' : 'val2'}, key_prefix='subspace_')
+ ... {'key1' : 'val1', 'key2' : 'val2'},
+ ... key_prefix='subspace_')
>>> len(notset_keys) == 0
True
- >>> mc.get_multi(['subspace_key1', 'subspace_key2']) == {'subspace_key1' : 'val1', 'subspace_key2' : 'val2'}
+ >>> mc.get_multi(['subspace_key1',
+ ... 'subspace_key2']) == {'subspace_key1': 'val1',
+ ... 'subspace_key2' : 'val2'}
True
Causes key 'subspace_key1' and 'subspace_key2' to be
@@ -715,44 +746,49 @@ class Client(local):
@param min_compress_len: The threshold length to kick in
auto-compression of the value using the zlib.compress()
- routine. If the value being cached is a string, then
- the length of the string is measured, else if the value
- is an object, then the length of the pickle result is
- measured. If the resulting attempt at compression yeilds
- a larger string than the input, then it is discarded. For
+ routine. If the value being cached is a string, then the
+ length of the string is measured, else if the value is an
+ object, then the length of the pickle result is
+ measured. If the resulting attempt at compression yeilds a
+ larger string than the input, then it is discarded. For
backwards compatability, this parameter defaults to 0,
indicating don't ever try to compress.
- @return: List of keys which failed to be stored [ memcache out of
- memory, etc. ].
- @rtype: list
- '''
+ @return: List of keys which failed to be stored [ memcache out
+ of memory, etc. ].
+ @rtype: list
+ '''
self._statlog('set_multi')
server_keys, prefixed_to_orig_key = self._map_and_prefix_keys(
- mapping.iterkeys(), key_prefix)
+ six.iterkeys(mapping), key_prefix)
# send out all requests on each server before reading anything
dead_servers = []
- notstored = [] # original keys.
+ notstored = [] # original keys.
- for server in server_keys.iterkeys():
+ for server in six.iterkeys(server_keys):
bigcmd = []
write = bigcmd.append
try:
- for key in server_keys[server]: # These are mangled keys
+ for key in server_keys[server]: # These are mangled keys
store_info = self._val_to_store_info(
- mapping[prefixed_to_orig_key[key]],
- min_compress_len)
+ mapping[prefixed_to_orig_key[key]],
+ min_compress_len)
if store_info:
- write("set %s %d %d %d\r\n%s\r\n" % (key, store_info[0],
- time, store_info[1], store_info[2]))
+ msg = "set %s %d %d %d\r\n%s\r\n"
+ write(msg % (key,
+ store_info[0],
+ time,
+ store_info[1],
+ store_info[2]))
else:
notstored.append(prefixed_to_orig_key[key])
server.send_cmds(''.join(bigcmd))
- except socket.error, msg:
- if isinstance(msg, tuple): msg = msg[1]
+ except socket.error as msg:
+ if isinstance(msg, tuple):
+ msg = msg[1]
server.mark_dead(msg)
dead_servers.append(server)
@@ -761,23 +797,28 @@ class Client(local):
del server_keys[server]
# short-circuit if there are no servers, just return all keys
- if not server_keys: return(mapping.keys())
+ if not server_keys:
+ return(mapping.keys())
- for server, keys in server_keys.iteritems():
+ for server, keys in six.iteritems(server_keys):
try:
for key in keys:
if server.readline() == 'STORED':
continue
else:
- notstored.append(prefixed_to_orig_key[key]) #un-mangle.
- except (_Error, socket.error), msg:
- if isinstance(msg, tuple): msg = msg[1]
+ # un-mangle.
+ notstored.append(prefixed_to_orig_key[key])
+ except (_Error, socket.error) as msg:
+ if isinstance(msg, tuple):
+ msg = msg[1]
server.mark_dead(msg)
return notstored
def _val_to_store_info(self, val, min_compress_len):
- """
- Transform val to a storable representation, returning a tuple of the flags, the length of the new value, and the new value itself.
+ """Transform val to a storable representation.
+
+ Returns a tuple of the flags, the length of the new value, and
+ the new value itself.
"""
flags = 0
if isinstance(val, str):
@@ -794,21 +835,23 @@ class Client(local):
min_compress_len = 0
else:
flags |= Client._FLAG_PICKLE
- file = StringIO()
+ file = io.StringIO()
if self.picklerIsKeyword:
- pickler = self.pickler(file, protocol = self.pickleProtocol)
+ pickler = self.pickler(file, protocol=self.pickleProtocol)
else:
pickler = self.pickler(file, self.pickleProtocol)
if self.persistent_id:
pickler.persistent_id = self.persistent_id
+ print(repr(val))
pickler.dump(val)
val = file.getvalue()
lv = len(val)
- # We should try to compress if min_compress_len > 0 and we could
- # import zlib and this string is longer than our min threshold.
- if min_compress_len and _supports_compress and lv > min_compress_len:
- comp_val = compress(val)
+ # We should try to compress if min_compress_len > 0 and we
+ # could import zlib and this string is longer than our min
+ # threshold.
+ if min_compress_len and lv > min_compress_len:
+ comp_val = zlib.compress(val)
# Only retain the result if the compression result is smaller
# than the original.
if len(comp_val) < lv:
@@ -817,11 +860,12 @@ class Client(local):
# silently do not store if value length exceeds maximum
if self.server_max_value_length != 0 and \
- len(val) > self.server_max_value_length: return(0)
+ len(val) > self.server_max_value_length:
+ return(0)
return (flags, len(val), val)
- def _set(self, cmd, key, val, time, min_compress_len = 0):
+ def _set(self, cmd, key, val, time, min_compress_len=0):
if self.do_check_key:
self.check_key(key)
server, key = self._get_server(key)
@@ -832,24 +876,28 @@ class Client(local):
self._statlog(cmd)
store_info = self._val_to_store_info(val, min_compress_len)
- if not store_info: return(0)
+ if not store_info:
+ return(0)
if cmd == 'cas':
if key not in self.cas_ids:
return self._set('set', key, val, time, min_compress_len)
fullcmd = "%s %s %d %d %d %d\r\n%s" % (
- cmd, key, store_info[0], time, store_info[1],
- self.cas_ids[key], store_info[2])
+ cmd, key, store_info[0], time, store_info[1],
+ self.cas_ids[key], store_info[2])
else:
fullcmd = "%s %s %d %d %d\r\n%s" % (
- cmd, key, store_info[0], time, store_info[1], store_info[2])
+ cmd, key, store_info[0],
+ time, store_info[1], store_info[2]
+ )
try:
server.send_cmd(fullcmd)
return(server.expect("STORED", raise_exception=True)
- == "STORED")
- except socket.error, msg:
- if isinstance(msg, tuple): msg = msg[1]
+ == "STORED")
+ except socket.error as msg:
+ if isinstance(msg, tuple):
+ msg = msg[1]
server.mark_dead(msg)
return 0
@@ -860,7 +908,7 @@ class Client(local):
try:
if server._get_socket():
return _unsafe_set()
- except (_ConnectionDeadError, socket.error), msg:
+ except (_ConnectionDeadError, socket.error) as msg:
server.mark_dead(msg)
return 0
@@ -879,13 +927,15 @@ class Client(local):
rkey = flags = rlen = cas_id = None
if cmd == 'gets':
- rkey, flags, rlen, cas_id, = self._expect_cas_value(server,
- raise_exception=True)
+ rkey, flags, rlen, cas_id, = self._expect_cas_value(
+ server, raise_exception=True
+ )
if rkey and self.cache_cas:
self.cas_ids[rkey] = cas_id
else:
- rkey, flags, rlen, = self._expectvalue(server,
- raise_exception=True)
+ rkey, flags, rlen, = self._expectvalue(
+ server, raise_exception=True
+ )
if not rkey:
return None
@@ -893,8 +943,9 @@ class Client(local):
value = self._recv_value(server, flags, rlen)
finally:
server.expect("END", raise_exception=True)
- except (_Error, socket.error), msg:
- if isinstance(msg, tuple): msg = msg[1]
+ except (_Error, socket.error) as msg:
+ if isinstance(msg, tuple):
+ msg = msg[1]
server.mark_dead(msg)
return None
@@ -908,7 +959,7 @@ class Client(local):
if server.connect():
return _unsafe_get()
return None
- except (_ConnectionDeadError, socket.error), msg:
+ except (_ConnectionDeadError, socket.error) as msg:
server.mark_dead(msg)
return None
@@ -927,54 +978,73 @@ class Client(local):
return self._get('gets', key)
def get_multi(self, keys, key_prefix=''):
- '''
- Retrieves multiple keys from the memcache doing just one query.
+ '''Retrieves multiple keys from the memcache doing just one query.
>>> success = mc.set("foo", "bar")
>>> success = mc.set("baz", 42)
- >>> mc.get_multi(["foo", "baz", "foobar"]) == {"foo": "bar", "baz": 42}
+ >>> mc.get_multi(["foo", "baz", "foobar"]) == {
+ ... "foo": "bar", "baz": 42
+ ... }
1
>>> mc.set_multi({'k1' : 1, 'k2' : 2}, key_prefix='pfx_') == []
1
- This looks up keys 'pfx_k1', 'pfx_k2', ... . Returned dict will just have unprefixed keys 'k1', 'k2'.
- >>> mc.get_multi(['k1', 'k2', 'nonexist'], key_prefix='pfx_') == {'k1' : 1, 'k2' : 2}
+ This looks up keys 'pfx_k1', 'pfx_k2', ... . Returned dict
+ will just have unprefixed keys 'k1', 'k2'.
+
+ >>> mc.get_multi(['k1', 'k2', 'nonexist'],
+ ... key_prefix='pfx_') == {'k1' : 1, 'k2' : 2}
1
- get_mult [ and L{set_multi} ] can take str()-ables like ints / longs as keys too. Such as your db pri key fields.
- They're rotored through str() before being passed off to memcache, with or without the use of a key_prefix.
- In this mode, the key_prefix could be a table name, and the key itself a db primary key number.
+ get_mult [ and L{set_multi} ] can take str()-ables like ints /
+ longs as keys too. Such as your db pri key fields. They're
+ rotored through str() before being passed off to memcache,
+ with or without the use of a key_prefix. In this mode, the
+ key_prefix could be a table name, and the key itself a db
+ primary key number.
- >>> mc.set_multi({42: 'douglass adams', 46 : 'and 2 just ahead of me'}, key_prefix='numkeys_') == []
+ >>> mc.set_multi({42: 'douglass adams',
+ ... 46: 'and 2 just ahead of me'},
+ ... key_prefix='numkeys_') == []
1
- >>> mc.get_multi([46, 42], key_prefix='numkeys_') == {42: 'douglass adams', 46 : 'and 2 just ahead of me'}
+ >>> mc.get_multi([46, 42], key_prefix='numkeys_') == {
+ ... 42: 'douglass adams',
+ ... 46: 'and 2 just ahead of me'
+ ... }
1
- This method is recommended over regular L{get} as it lowers the number of
- total packets flying around your network, reducing total latency, since
- your app doesn't have to wait for each round-trip of L{get} before sending
- the next one.
+ This method is recommended over regular L{get} as it lowers
+ the number of total packets flying around your network,
+ reducing total latency, since your app doesn't have to wait
+ for each round-trip of L{get} before sending the next one.
See also L{set_multi}.
@param keys: An array of keys.
- @param key_prefix: A string to prefix each key when we communicate with memcache.
- Facilitates pseudo-namespaces within memcache. Returned dictionary keys will not have this prefix.
- @return: A dictionary of key/value pairs that were available. If key_prefix was provided, the keys in the retured dictionary will not have it present.
+ @param key_prefix: A string to prefix each key when we
+ communicate with memcache. Facilitates pseudo-namespaces
+ within memcache. Returned dictionary keys will not have this
+ prefix.
+
+ @return: A dictionary of key/value pairs that were
+ available. If key_prefix was provided, the keys in the retured
+ dictionary will not have it present.
'''
self._statlog('get_multi')
- server_keys, prefixed_to_orig_key = self._map_and_prefix_keys(keys, key_prefix)
+ server_keys, prefixed_to_orig_key = self._map_and_prefix_keys(
+ keys, key_prefix)
# send out all requests on each server before reading anything
dead_servers = []
- for server in server_keys.iterkeys():
+ for server in six.iterkeys(server_keys):
try:
server.send_cmd("get %s" % " ".join(server_keys[server]))
- except socket.error, msg:
- if isinstance(msg, tuple): msg = msg[1]
+ except socket.error as msg:
+ if isinstance(msg, tuple):
+ msg = msg[1]
server.mark_dead(msg)
dead_servers.append(server)
@@ -983,7 +1053,7 @@ class Client(local):
del server_keys[server]
retvals = {}
- for server in server_keys.iterkeys():
+ for server in six.iterkeys(server_keys):
try:
line = server.readline()
while line and line != 'END':
@@ -991,10 +1061,12 @@ class Client(local):
# Bo Yang reports that this can sometimes be None
if rkey is not None:
val = self._recv_value(server, flags, rlen)
- retvals[prefixed_to_orig_key[rkey]] = val # un-prefix returned key.
+ # un-prefix returned key.
+ retvals[prefixed_to_orig_key[rkey]] = val
line = server.readline()
- except (_Error, socket.error), msg:
- if isinstance(msg, tuple): msg = msg[1]
+ except (_Error, socket.error) as msg:
+ if isinstance(msg, tuple):
+ msg = msg[1]
server.mark_dead(msg)
return retvals
@@ -1021,19 +1093,19 @@ class Client(local):
return (None, None, None)
def _recv_value(self, server, flags, rlen):
- rlen += 2 # include \r\n
+ rlen += 2 # include \r\n
buf = server.recv(rlen)
if len(buf) != rlen:
raise _Error("received %d bytes when expecting %d"
- % (len(buf), rlen))
+ % (len(buf), rlen))
if len(buf) == rlen:
buf = buf[:-2] # strip \r\n
if flags & Client._FLAG_COMPRESSED:
- buf = decompress(buf)
+ buf = zlib.decompress(buf)
- if flags == 0 or flags == Client._FLAG_COMPRESSED:
+ if flags == 0 or flags == Client._FLAG_COMPRESSED:
# Either a bare string or a compressed string now decompressed...
val = buf
elif flags & Client._FLAG_INTEGER:
@@ -1042,12 +1114,12 @@ class Client(local):
val = long(buf)
elif flags & Client._FLAG_PICKLE:
try:
- file = StringIO(buf)
+ file = io.StringIO(buf)
unpickler = self.unpickler(file)
if self.persistent_load:
unpickler.persistent_load = self.persistent_load
val = unpickler.load()
- except Exception, e:
+ except Exception as e:
self.debuglog('Pickle error: %s\n' % e)
return None
else:
@@ -1065,24 +1137,25 @@ class Client(local):
Is not a string (Raises MemcachedKeyError)
Is None (Raises MemcachedKeyError)
"""
- if isinstance(key, tuple): key = key[1]
+ if isinstance(key, tuple):
+ key = key[1]
if not key:
raise Client.MemcachedKeyNoneError("Key is None")
- if isinstance(key, unicode):
- raise Client.MemcachedStringEncodingError(
- "Keys must be str()'s, not unicode. Convert your unicode "
- "strings using mystring.encode(charset)!")
- if not isinstance(key, str):
+ if isinstance(key, six.text_type):
+ key = key.encode('ascii')
+ if not isinstance(key, six.binary_type):
raise Client.MemcachedKeyTypeError("Key must be str()'s")
- if isinstance(key, basestring):
+ if isinstance(key, six.string_types):
if self.server_max_key_length != 0 and \
- len(key) + key_extra_len > self.server_max_key_length:
- raise Client.MemcachedKeyLengthError("Key length is > %s"
- % self.server_max_key_length)
+ len(key) + key_extra_len > self.server_max_key_length:
+ raise Client.MemcachedKeyLengthError(
+ "Key length is > %s" % self.server_max_key_length
+ )
if not valid_key_chars_re.match(key):
raise Client.MemcachedKeyCharacterError(
- "Control characters not allowed")
+ "Control characters not allowed")
+
class _Host(object):
@@ -1101,11 +1174,12 @@ class _Host(object):
m = re.match(r'^(?P<proto>unix):(?P<path>.*)$', host)
if not m:
m = re.match(r'^(?P<proto>inet6):'
- r'\[(?P<host>[^\[\]]+)\](:(?P<port>[0-9]+))?$', host)
+ r'\[(?P<host>[^\[\]]+)\](:(?P<port>[0-9]+))?$', host)
if not m:
m = re.match(r'^(?P<proto>inet):'
- r'(?P<host>[^:]+)(:(?P<port>[0-9]+))?$', host)
- if not m: m = re.match(r'^(?P<host>[^:]+)(:(?P<port>[0-9]+))?$', host)
+ r'(?P<host>[^:]+)(:(?P<port>[0-9]+))?$', host)
+ if not m:
+ m = re.match(r'^(?P<host>[^:]+)(:(?P<port>[0-9]+))?$', host)
if not m:
raise ValueError('Unable to parse connection string: "%s"' % host)
@@ -1117,12 +1191,12 @@ class _Host(object):
self.family = socket.AF_INET6
self.ip = hostData['host']
self.port = int(hostData.get('port') or 11211)
- self.address = ( self.ip, self.port )
+ self.address = (self.ip, self.port)
else:
self.family = socket.AF_INET
self.ip = hostData['host']
self.port = int(hostData.get('port') or 11211)
- self.address = ( self.ip, self.port )
+ self.address = (self.ip, self.port)
self.deaduntil = 0
self.socket = None
@@ -1158,14 +1232,16 @@ class _Host(object):
if self.socket:
return self.socket
s = socket.socket(self.family, socket.SOCK_STREAM)
- if hasattr(s, 'settimeout'): s.settimeout(self.socket_timeout)
+ if hasattr(s, 'settimeout'):
+ s.settimeout(self.socket_timeout)
try:
s.connect(self.address)
- except socket.timeout, msg:
+ except socket.timeout as msg:
self.mark_dead("connect: %s" % msg)
return None
- except socket.error, msg:
- if isinstance(msg, tuple): msg = msg[1]
+ except socket.error as msg:
+ if isinstance(msg, tuple):
+ msg = msg[1]
self.mark_dead("connect: %s" % msg[1])
return None
self.socket = s
@@ -1184,13 +1260,14 @@ class _Host(object):
self.socket.sendall(cmd + '\r\n')
def send_cmds(self, cmds):
- """ cmds already has trailing \r\n's applied """
+ """cmds already has trailing \r\n's applied."""
self.socket.sendall(cmds)
def readline(self, raise_exception=False):
- """Read a line and return it. If "raise_exception" is set,
- raise _ConnectionDeadError if the read fails, otherwise return
- an empty string.
+ """Read a line and return it.
+
+ If "raise_exception" is set, raise _ConnectionDeadError if the
+ read fails, otherwise return an empty string.
"""
buf = self.buffer
if self.socket:
@@ -1212,14 +1289,14 @@ class _Host(object):
return ''
buf += data
- self.buffer = buf[index+2:]
+ self.buffer = buf[index + 2:]
return buf[:index]
def expect(self, text, raise_exception=False):
line = self.readline(raise_exception)
if line != text:
self.debuglog("while expecting '%s', got unexpected response '%s'"
- % (text, line))
+ % (text, line))
return line
def recv(self, rlen):
@@ -1229,8 +1306,8 @@ class _Host(object):
foo = self_socket_recv(max(rlen - len(buf), 4096))
buf += foo
if not foo:
- raise _Error( 'Read %d bytes, expecting %d, '
- 'read returned 0 length bytes' % ( len(buf), rlen ))
+ raise _Error('Read %d bytes, expecting %d, '
+ 'read returned 0 length bytes' % (len(buf), rlen))
self.buffer = buf[rlen:]
return buf[:rlen]
@@ -1252,7 +1329,8 @@ class _Host(object):
def _doctest():
- import doctest, memcache
+ import doctest
+ import memcache
servers = ["127.0.0.1:11211"]
mc = Client(servers, debug=1)
globs = {"mc": mc}
@@ -1260,9 +1338,9 @@ def _doctest():
if __name__ == "__main__":
failures = 0
- print "Testing docstrings..."
+ print("Testing docstrings...")
_doctest()
- print "Running tests:"
+ print("Running tests:")
print
serverList = [["127.0.0.1:11211"]]
if '--do-unix' in sys.argv:
@@ -1275,24 +1353,28 @@ if __name__ == "__main__":
if not isinstance(val, basestring):
return "%s (%s)" % (val, type(val))
return "%s" % val
+
def test_setget(key, val):
global failures
- print "Testing set/get {'%s': %s} ..." % (to_s(key), to_s(val)),
+ print("Testing set/get {'%s': %s} ..." % (to_s(key), to_s(val)),)
mc.set(key, val)
newval = mc.get(key)
if newval == val:
- print "OK"
+ print("OK")
return 1
else:
- print "FAIL"; failures = failures + 1
+ print("FAIL")
+ failures = failures + 1
return 0
-
class FooStruct(object):
+
def __init__(self):
self.bar = "baz"
+
def __str__(self):
return "A FooStruct"
+
def __eq__(self, other):
if isinstance(other, FooStruct):
return self.bar == other.bar
@@ -1300,139 +1382,141 @@ if __name__ == "__main__":
test_setget("a_string", "some random string")
test_setget("an_integer", 42)
- if test_setget("long", long(1<<30)):
- print "Testing delete ...",
+ if test_setget("long", long(1 << 30)):
+ print("Testing delete ...",)
if mc.delete("long"):
- print "OK"
+ print("OK")
else:
- print "FAIL"; failures = failures + 1
- print "Checking results of delete ..."
- if mc.get("long") == None:
- print "OK"
+ print("FAIL")
+ failures = failures + 1
+ print("Checking results of delete ...")
+ if mc.get("long") is None:
+ print("OK")
else:
- print "FAIL"; failures = failures + 1
- print "Testing get_multi ...",
- print mc.get_multi(["a_string", "an_integer"])
-
- # removed from the protocol
- #if test_setget("timed_delete", 'foo'):
- # print "Testing timed delete ...",
- # if mc.delete("timed_delete", 1):
- # print "OK"
- # else:
- # print "FAIL"; failures = failures + 1
- # print "Checking results of timed delete ..."
- # if mc.get("timed_delete") == None:
- # print "OK"
- # else:
- # print "FAIL"; failures = failures + 1
-
- print "Testing get(unknown value) ...",
- print to_s(mc.get("unknown_value"))
+ print("FAIL")
+ failures = failures + 1
+ print("Testing get_multi ...",)
+ print(mc.get_multi(["a_string", "an_integer"]))
+ print("Testing get(unknown value) ...",)
+ print(to_s(mc.get("unknown_value")))
f = FooStruct()
test_setget("foostruct", f)
- print "Testing incr ...",
+ print("Testing incr ...",)
x = mc.incr("an_integer", 1)
if x == 43:
- print "OK"
+ print("OK")
else:
- print "FAIL"; failures = failures + 1
+ print("FAIL")
+ failures = failures + 1
- print "Testing decr ...",
+ print("Testing decr ...",)
x = mc.decr("an_integer", 1)
if x == 42:
- print "OK"
+ print("OK")
else:
- print "FAIL"; failures = failures + 1
+ print("FAIL")
+ failures = failures + 1
sys.stdout.flush()
# sanity tests
- print "Testing sending spaces...",
+ print("Testing sending spaces...",)
sys.stdout.flush()
try:
x = mc.set("this has spaces", 1)
- except Client.MemcachedKeyCharacterError, msg:
- print "OK"
+ except Client.MemcachedKeyCharacterError as msg:
+ print("OK")
else:
- print "FAIL"; failures = failures + 1
+ print("FAIL")
+ failures = failures + 1
- print "Testing sending control characters...",
+ print("Testing sending control characters...",)
try:
x = mc.set("this\x10has\x11control characters\x02", 1)
- except Client.MemcachedKeyCharacterError, msg:
- print "OK"
+ except Client.MemcachedKeyCharacterError as msg:
+ print("OK")
else:
- print "FAIL"; failures = failures + 1
+ print("FAIL")
+ failures = failures + 1
- print "Testing using insanely long key...",
+ print("Testing using insanely long key...",)
try:
- x = mc.set('a'*SERVER_MAX_KEY_LENGTH, 1)
- except Client.MemcachedKeyLengthError, msg:
- print "FAIL"; failures = failures + 1
+ x = mc.set('a' * SERVER_MAX_KEY_LENGTH, 1)
+ except Client.MemcachedKeyLengthError as msg:
+ print("FAIL")
+ failures = failures + 1
else:
- print "OK"
+ print("OK")
try:
- x = mc.set('a'*SERVER_MAX_KEY_LENGTH + 'a', 1)
- except Client.MemcachedKeyLengthError, msg:
- print "OK"
+ x = mc.set('a' * SERVER_MAX_KEY_LENGTH + 'a', 1)
+ except Client.MemcachedKeyLengthError as msg:
+ print("OK")
else:
- print "FAIL"; failures = failures + 1
+ failures = failures + 1
+ print("FAIL")
- print "Testing sending a unicode-string key...",
+ print("Testing sending a unicode-string key...",)
try:
x = mc.set(unicode('keyhere'), 1)
- except Client.MemcachedStringEncodingError, msg:
- print "OK",
+ except Client.MemcachedStringEncodingError as msg:
+ print("OK",)
else:
- print "FAIL",; failures = failures + 1
+ failures = failures + 1
+ print("FAIL")
try:
- x = mc.set((unicode('a')*SERVER_MAX_KEY_LENGTH).encode('utf-8'), 1)
- except:
- print "FAIL",; failures = failures + 1
+ x = mc.set((unicode('a') * SERVER_MAX_KEY_LENGTH)
+ .encode('utf-8'), 1)
+ except Exception:
+ failures = failures + 1
+ print("FAIL")
else:
- print "OK",
+ print("OK",)
import pickle
s = pickle.loads('V\\u4f1a\np0\n.')
try:
- x = mc.set((s*SERVER_MAX_KEY_LENGTH).encode('utf-8'), 1)
+ x = mc.set((s * SERVER_MAX_KEY_LENGTH).encode('utf-8'), 1)
except Client.MemcachedKeyLengthError:
- print "OK"
+ print("OK")
else:
- print "FAIL"; failures = failures + 1
-
- print "Testing using a value larger than the memcached value limit..."
- print 'NOTE: "MemCached: while expecting[...]" is normal...'
- x = mc.set('keyhere', 'a'*SERVER_MAX_VALUE_LENGTH)
- if mc.get('keyhere') == None:
- print "OK",
+ failures = failures + 1
+ print("FAIL")
+
+ print("Testing using a value larger than the memcached value limit...")
+ print('NOTE: "MemCached: while expecting[...]" is normal...')
+ x = mc.set('keyhere', 'a' * SERVER_MAX_VALUE_LENGTH)
+ if mc.get('keyhere') is None:
+ print("OK",)
else:
- print "FAIL",; failures = failures + 1
- x = mc.set('keyhere', 'a'*SERVER_MAX_VALUE_LENGTH + 'aaa')
- if mc.get('keyhere') == None:
- print "OK"
+ failures = failures + 1
+ print("FAIL")
+ x = mc.set('keyhere', 'a' * SERVER_MAX_VALUE_LENGTH + 'aaa')
+ if mc.get('keyhere') is None:
+ print("OK")
else:
- print "FAIL"; failures = failures + 1
+ failures = failures + 1
+ print("FAIL")
- print "Testing set_multi() with no memcacheds running",
+ print("Testing set_multi() with no memcacheds running",)
mc.disconnect_all()
- errors = mc.set_multi({'keyhere' : 'a', 'keythere' : 'b'})
+ errors = mc.set_multi({'keyhere': 'a', 'keythere': 'b'})
if errors != []:
- print "FAIL"; failures = failures + 1
+ failures = failures + 1
+ print("FAIL")
else:
- print "OK"
+ print("OK")
- print "Testing delete_multi() with no memcacheds running",
+ print("Testing delete_multi() with no memcacheds running",)
mc.disconnect_all()
- ret = mc.delete_multi({'keyhere' : 'a', 'keythere' : 'b'})
+ ret = mc.delete_multi({'keyhere': 'a', 'keythere': 'b'})
if ret != 1:
- print "FAIL"; failures = failures + 1
+ failures = failures + 1
+ print("FAIL")
else:
- print "OK"
+ print("OK")
if failures > 0:
- print '*** THERE WERE FAILED TESTS'
+ print('*** THERE WERE FAILED TESTS')
sys.exit(1)
sys.exit(0)
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..ffe2fce
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1 @@
+six
diff --git a/setup.py b/setup.py
index f66b227..96f5787 100644
--- a/setup.py
+++ b/setup.py
@@ -1,10 +1,10 @@
#!/usr/bin/env python
-from setuptools import setup
-import memcache
+from setuptools import setup # noqa
+
setup(name="python-memcached",
- version=memcache.__version__,
+ version="1.53",
description="Pure python memcached client",
long_description=open("README.md").read(),
author="Evan Martin",
@@ -14,13 +14,13 @@ setup(name="python-memcached",
url="http://www.tummy.com/Community/software/python-memcached/",
download_url="ftp://ftp.tummy.com/pub/python-memcached/",
py_modules=["memcache"],
+ requirements=open('requirements.txt').read().split(),
classifiers=[
- "Development Status :: 5 - Production/Stable",
- "Intended Audience :: Developers",
- "License :: OSI Approved :: Python Software Foundation License",
- "Operating System :: OS Independent",
- "Programming Language :: Python",
- "Topic :: Internet",
- "Topic :: Software Development :: Libraries :: Python Modules",
- ])
-
+ "Development Status :: 5 - Production/Stable",
+ "Intended Audience :: Developers",
+ "License :: OSI Approved :: Python Software Foundation License",
+ "Operating System :: OS Independent",
+ "Programming Language :: Python",
+ "Topic :: Internet",
+ "Topic :: Software Development :: Libraries :: Python Modules",
+ ])
diff --git a/test-requirements.txt b/test-requirements.txt
new file mode 100644
index 0000000..8f21390
--- /dev/null
+++ b/test-requirements.txt
@@ -0,0 +1,3 @@
+nose
+coverage
+hacking
diff --git a/tests/Makefile b/tests/Makefile
deleted file mode 100644
index a844aa4..0000000
--- a/tests/Makefile
+++ /dev/null
@@ -1,7 +0,0 @@
-TESTS = $(wildcard test_*.py)
-
-test:
- @- $(foreach TEST,$(TESTS), \
- echo === Running test: $(TEST); \
- python $(TEST); \
- )
diff --git a/tests/test_setmulti.py b/tests/test_setmulti.py
index 5947639..f10cb59 100644
--- a/tests/test_setmulti.py
+++ b/tests/test_setmulti.py
@@ -24,16 +24,16 @@ class test_Memcached_Set_Multi(unittest.TestCase):
class FakeSocket(object):
def __init__(self, *args):
if DEBUG:
- print 'FakeSocket{0!r}'.format(args)
+ print('FakeSocket{0!r}'.format(args))
self._recv_chunks = list(RECV_CHUNKS)
def connect(self, *args):
if DEBUG:
- print 'FakeSocket.connect{0!r}'.format(args)
+ print('FakeSocket.connect{0!r}'.format(args))
def sendall(self, *args):
if DEBUG:
- print 'FakeSocket.sendall{0!r}'.format(args)
+ print('FakeSocket.sendall{0!r}'.format(args))
def recv(self, *args):
if self._recv_chunks:
@@ -41,12 +41,12 @@ class test_Memcached_Set_Multi(unittest.TestCase):
else:
data = ''
if DEBUG:
- print 'FakeSocket.recv{0!r} -> {1!r}'.format(args, data)
+ print('FakeSocket.recv{0!r} -> {1!r}'.format(args, data))
return data
def close(self):
if DEBUG:
- print 'FakeSocket.close()'
+ print('FakeSocket.close()')
self.old_socket = socket.socket
socket.socket = FakeSocket
@@ -62,7 +62,7 @@ class test_Memcached_Set_Multi(unittest.TestCase):
self.assertEqual(sorted(bad_keys), ['bar', 'foo'])
if DEBUG:
- print 'set_multi({0!r}) -> {1!r}'.format(mapping, bad_keys)
+ print('set_multi({0!r}) -> {1!r}'.format(mapping, bad_keys))
if __name__ == '__main__':
unittest.main()
diff --git a/tox.ini b/tox.ini
new file mode 100644
index 0000000..553b75c
--- /dev/null
+++ b/tox.ini
@@ -0,0 +1,24 @@
+[tox]
+minversion = 1.6
+envlist = py26,py27,py33,pypy,pep8
+skipsdist = True
+
+[testenv]
+usedevelop = True
+# Customize pip command, add -U to force updates.
+install_command = pip install -U {opts} {packages}
+deps = -r{toxinidir}/requirements.txt
+ -r{toxinidir}/test-requirements.txt
+commands = nosetests {posargs}
+
+[tox:jenkins]
+downloadcache = ~/cache/pip
+
+[testenv:pep8]
+commands = flake8
+
+[testenv:cover]
+commands = nosetests --with-coverage {posargs}
+
+[flake8]
+exclude = .venv*,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*.egg,.update-venv