diff options
author | Sean Reifschneider <jafo@tummy.com> | 2014-03-25 22:45:16 -0600 |
---|---|---|
committer | Sean Reifschneider <jafo@tummy.com> | 2014-03-25 22:45:16 -0600 |
commit | 179eeca44bfae70cfc4d7f7d2fd10d808f47416f (patch) | |
tree | 8c98b8bfaa3e0575abf7bbb195b2c9f7699f2be7 | |
parent | e3468928c53588e66bfabcbe6fde639222de02b3 (diff) | |
parent | 853040beb9d2b709e67cae77fffd08dceab8b61a (diff) | |
download | python-memcached-179eeca44bfae70cfc4d7f7d2fd10d808f47416f.tar.gz |
Merge branch 'py33_fixes' of https://github.com/cabrera/python-memcached into cabrera-py33_fixes
Conflicts:
README.md
memcache.py
tests/test_setmulti.py
-rw-r--r-- | .gitignore | 2 | ||||
-rw-r--r-- | README.md | 13 | ||||
-rw-r--r-- | memcache.py | 950 | ||||
-rw-r--r-- | requirements.txt | 1 | ||||
-rw-r--r-- | setup.py | 24 | ||||
-rw-r--r-- | test-requirements.txt | 3 | ||||
-rw-r--r-- | tests/Makefile | 7 | ||||
-rw-r--r-- | tests/test_setmulti.py | 12 | ||||
-rw-r--r-- | tox.ini | 24 |
9 files changed, 578 insertions, 458 deletions
@@ -1,3 +1,5 @@ *.pyc /dist /python_memcached.egg-info +.tox +.coverage @@ -1,6 +1,8 @@ [![Build Status](https://travis-ci.org/linsomniac/python-memcached.svg)](https://travis-ci.org/linsomniac/python-memcached) +## Overview + This software is a 100% Python interface to the memcached memory cache daemon. It is the client side software which allows storing values in one or more, possibly remote, memcached servers. Search google for @@ -17,3 +19,14 @@ Please report issues and submit code changes to the github repository at: For changes prior to 2013-03-26, see the old Launchpad repository at: Historic issues: https://launchpad.net/python-memcached + +## Testing + +Test patches locally and easily by running tox: + + pip install tox + tox -e py27 + +Test for style by running tox: + + tox -e pep8 diff --git a/memcache.py b/memcache.py index 769610e..ea96b40 100644 --- a/memcache.py +++ b/memcache.py @@ -1,12 +1,12 @@ #!/usr/bin/env python -""" -client module for memcached (memory cache daemon) +"""client module for memcached (memory cache daemon) Overview ======== -See U{the MemCached homepage<http://www.danga.com/memcached>} for more about memcached. +See U{the MemCached homepage<http://www.danga.com/memcached>} for more +about memcached. Usage summary ============= @@ -22,7 +22,8 @@ This should give you a feel for how this module operates:: mc.set("another_key", 3) mc.delete("another_key") - mc.set("key", "1") # note that the key used for incr/decr must be a string. + mc.set("key", "1") # note that the key used for incr/decr must be + # a string. mc.incr("key") mc.decr("key") @@ -41,57 +42,50 @@ Detailed Documentation ====================== More detailed documentation is available in the L{Client} class. + """ -import sys -import socket -import time +from __future__ import print_function +import binascii +import io import os +import pickle import re -try: - import cPickle as pickle -except ImportError: - import pickle +import socket +import sys +import threading +import time +import zlib + +import six + -from binascii import crc32 # zlib version is not cross-platform def cmemcache_hash(key): - return((((crc32(key) & 0xffffffff) >> 16) & 0x7fff) or 1) + return ((((binascii.crc32(key.encode('ascii')) & 0xffffffff) >> 16) & 0x7fff) or 1) serverHashFunction = cmemcache_hash + def useOldServerHashFunction(): """Use the old python-memcache server hash function.""" global serverHashFunction - serverHashFunction = crc32 - -try: - from zlib import compress, decompress - _supports_compress = True -except ImportError: - _supports_compress = False - # quickly define a decompress just in case we recv compressed data. - def decompress(val): - raise _Error("received compressed data but I don't support compression (import error)") - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO + serverHashFunction = binascii.crc32 + valid_key_chars_re = re.compile('[\x21-\x7e\x80-\xff]+$') # Original author: Evan Martin of Danga Interactive -__author__ = "Sean Reifschneider <jafo-memcached@tummy.com>" +__author__ = "Sean Reifschneider <jafo-memcached@tummy.com>" __version__ = "1.53" __copyright__ = "Copyright (C) 2003 Danga Interactive" # http://en.wikipedia.org/wiki/Python_Software_Foundation_License -__license__ = "Python Software Foundation License" +__license__ = "Python Software Foundation License" SERVER_MAX_KEY_LENGTH = 250 -# Storing values larger than 1MB requires recompiling memcached. If you do, -# this value can be changed by doing "memcache.SERVER_MAX_VALUE_LENGTH = N" -# after importing this module. -SERVER_MAX_VALUE_LENGTH = 1024*1024 +# Storing values larger than 1MB requires recompiling memcached. If +# you do, this value can be changed by doing +# "memcache.SERVER_MAX_VALUE_LENGTH = N" after importing this module. +SERVER_MAX_VALUE_LENGTH = 1024 * 1024 class _Error(Exception): @@ -102,58 +96,57 @@ class _ConnectionDeadError(Exception): pass -try: - # Only exists in Python 2.4+ - from threading import local -except ImportError: - # TODO: add the pure-python local implementation - class local(object): - pass - - _DEAD_RETRY = 30 # number of seconds before retrying a dead server. -_SOCKET_TIMEOUT = 3 # number of seconds before sockets timeout. +_SOCKET_TIMEOUT = 3 # number of seconds before sockets timeout. -class Client(local): - """ - Object representing a pool of memcache servers. +class Client(threading.local): + """Object representing a pool of memcache servers. See L{memcache} for an overview. In all cases where a key is used, the key can be either: 1. A simple hashable type (string, integer, etc.). - 2. A tuple of C{(hashvalue, key)}. This is useful if you want to avoid - making this module calculate a hash value. You may prefer, for - example, to keep all of a given user's objects on the same memcache - server, so you could use the user's unique id as the hash value. + 2. A tuple of C{(hashvalue, key)}. This is useful if you want + to avoid making this module calculate a hash value. You may + prefer, for example, to keep all of a given user's objects on + the same memcache server, so you could use the user's unique + id as the hash value. + - @group Setup: __init__, set_servers, forget_dead_hosts, disconnect_all, debuglog + @group Setup: __init__, set_servers, forget_dead_hosts, + disconnect_all, debuglog @group Insertion: set, add, replace, set_multi @group Retrieval: get, get_multi @group Integers: incr, decr @group Removal: delete, delete_multi - @sort: __init__, set_servers, forget_dead_hosts, disconnect_all, debuglog,\ - set, set_multi, add, replace, get, get_multi, incr, decr, delete, delete_multi + @sort: __init__, set_servers, forget_dead_hosts, disconnect_all, + debuglog,\ set, set_multi, add, replace, get, get_multi, + incr, decr, delete, delete_multi """ - _FLAG_PICKLE = 1<<0 - _FLAG_INTEGER = 1<<1 - _FLAG_LONG = 1<<2 - _FLAG_COMPRESSED = 1<<3 + _FLAG_PICKLE = 1 << 0 + _FLAG_INTEGER = 1 << 1 + _FLAG_LONG = 1 << 2 + _FLAG_COMPRESSED = 1 << 3 _SERVER_RETRIES = 10 # how many times to try finding a free server. # exceptions for Client class MemcachedKeyError(Exception): pass + class MemcachedKeyLengthError(MemcachedKeyError): pass + class MemcachedKeyCharacterError(MemcachedKeyError): pass + class MemcachedKeyNoneError(MemcachedKeyError): pass + class MemcachedKeyTypeError(MemcachedKeyError): pass + class MemcachedStringEncodingError(Exception): pass @@ -162,43 +155,51 @@ class Client(local): pload=None, pid=None, server_max_key_length=None, server_max_value_length=None, dead_retry=_DEAD_RETRY, socket_timeout=_SOCKET_TIMEOUT, - cache_cas = False, flush_on_reconnect=0, check_keys=True): - """ - Create a new Client object with the given list of servers. + cache_cas=False, flush_on_reconnect=0, check_keys=True): + """Create a new Client object with the given list of servers. @param servers: C{servers} is passed to L{set_servers}. - @param debug: whether to display error messages when a server can't be - contacted. - @param pickleProtocol: number to mandate protocol used by (c)Pickle. - @param pickler: optional override of default Pickler to allow subclassing. - @param unpickler: optional override of default Unpickler to allow subclassing. - @param pload: optional persistent_load function to call on pickle loading. - Useful for cPickle since subclassing isn't allowed. - @param pid: optional persistent_id function to call on pickle storing. - Useful for cPickle since subclassing isn't allowed. - @param dead_retry: number of seconds before retrying a blacklisted - server. Default to 30 s. - @param socket_timeout: timeout in seconds for all calls to a server. Defaults - to 3 seconds. - @param cache_cas: (default False) If true, cas operations will be - cached. WARNING: This cache is not expired internally, if you have - a long-running process you will need to expire it manually via - client.reset_cas(), or the cache can grow unlimited. + @param debug: whether to display error messages when a server + can't be contacted. + @param pickleProtocol: number to mandate protocol used by + (c)Pickle. + @param pickler: optional override of default Pickler to allow + subclassing. + @param unpickler: optional override of default Unpickler to + allow subclassing. + @param pload: optional persistent_load function to call on + pickle loading. Useful for cPickle since subclassing isn't + allowed. + @param pid: optional persistent_id function to call on pickle + storing. Useful for cPickle since subclassing isn't allowed. + @param dead_retry: number of seconds before retrying a + blacklisted server. Default to 30 s. + @param socket_timeout: timeout in seconds for all calls to a + server. Defaults to 3 seconds. + @param cache_cas: (default False) If true, cas operations will + be cached. WARNING: This cache is not expired internally, if + you have a long-running process you will need to expire it + manually via client.reset_cas(), or the cache can grow + unlimited. @param server_max_key_length: (default SERVER_MAX_KEY_LENGTH) Data that is larger than this will not be sent to the server. - @param server_max_value_length: (default SERVER_MAX_VALUE_LENGTH) - Data that is larger than this will not be sent to the server. - @param flush_on_reconnect: optional flag which prevents a scenario that - can cause stale data to be read: If there's more than one memcached - server and the connection to one is interrupted, keys that mapped to - that server will get reassigned to another. If the first server comes - back, those keys will map to it again. If it still has its data, get()s - can read stale data that was overwritten on another server. This flag - is off by default for backwards compatibility. - @param check_keys: (default True) If True, the key is checked to - ensure it is the correct length and composed of the right characters. + @param server_max_value_length: (default + SERVER_MAX_VALUE_LENGTH) Data that is larger than this will + not be sent to the server. + @param flush_on_reconnect: optional flag which prevents a + scenario that can cause stale data to be read: If there's more + than one memcached server and the connection to one is + interrupted, keys that mapped to that server will get + reassigned to another. If the first server comes back, those + keys will map to it again. If it still has its data, get()s + can read stale data that was overwritten on another + server. This flag is off by default for backwards + compatibility. + @param check_keys: (default True) If True, the key is checked + to ensure it is the correct length and composed of the right + characters. """ - local.__init__(self) + super(Client, self).__init__() self.debug = debug self.dead_retry = dead_retry self.socket_timeout = socket_timeout @@ -223,69 +224,73 @@ class Client(local): self.server_max_value_length = SERVER_MAX_VALUE_LENGTH # figure out the pickler style - file = StringIO() + file = io.StringIO() try: - pickler = self.pickler(file, protocol = self.pickleProtocol) + pickler = self.pickler(file, protocol=self.pickleProtocol) self.picklerIsKeyword = True except TypeError: self.picklerIsKeyword = False def reset_cas(self): - """ - Reset the cas cache. This is only used if the Client() object - was created with "cache_cas=True". If used, this cache does not - expire internally, so it can grow unbounded if you do not clear it + """Reset the cas cache. + + This is only used if the Client() object was created with + "cache_cas=True". If used, this cache does not expire + internally, so it can grow unbounded if you do not clear it yourself. """ self.cas_ids = {} - def set_servers(self, servers): - """ - Set the pool of servers used by this client. + """Set the pool of servers used by this client. @param servers: an array of servers. Servers can be passed in two forms: - 1. Strings of the form C{"host:port"}, which implies a default weight of 1. - 2. Tuples of the form C{("host:port", weight)}, where C{weight} is - an integer weight value. + 1. Strings of the form C{"host:port"}, which implies a + default weight of 1. + 2. Tuples of the form C{("host:port", weight)}, where + C{weight} is an integer weight value. + """ self.servers = [_Host(s, self.debug, dead_retry=self.dead_retry, - socket_timeout=self.socket_timeout, - flush_on_reconnect=self.flush_on_reconnect) - for s in servers] + socket_timeout=self.socket_timeout, + flush_on_reconnect=self.flush_on_reconnect) + for s in servers] self._init_buckets() - def get_stats(self, stat_args = None): - '''Get statistics from each of the servers. + def get_stats(self, stat_args=None): + """Get statistics from each of the servers. @param stat_args: Additional arguments to pass to the memcache "stats" command. - @return: A list of tuples ( server_identifier, stats_dictionary ). - The dictionary contains a number of name/value pairs specifying - the name of the status field and the string value associated with - it. The values are not converted from strings. - ''' + @return: A list of tuples ( server_identifier, + stats_dictionary ). The dictionary contains a number of + name/value pairs specifying the name of the status field + and the string value associated with it. The values are + not converted from strings. + """ data = [] for s in self.servers: - if not s.connect(): continue + if not s.connect(): + continue if s.family == socket.AF_INET: - name = '%s:%s (%s)' % ( s.ip, s.port, s.weight ) + name = '%s:%s (%s)' % (s.ip, s.port, s.weight) elif s.family == socket.AF_INET6: - name = '[%s]:%s (%s)' % ( s.ip, s.port, s.weight ) + name = '[%s]:%s (%s)' % (s.ip, s.port, s.weight) else: - name = 'unix:%s (%s)' % ( s.address, s.weight ) + name = 'unix:%s (%s)' % (s.address, s.weight) if not stat_args: s.send_cmd('stats') else: s.send_cmd('stats ' + stat_args) serverData = {} - data.append(( name, serverData )) + data.append((name, serverData)) readline = s.readline while 1: line = readline() - if not line or line.strip() == 'END': break + if not line or line.strip() == 'END': + break stats = line.split(' ', 2) serverData[stats[1]] = stats[2] @@ -294,24 +299,26 @@ class Client(local): def get_slabs(self): data = [] for s in self.servers: - if not s.connect(): continue + if not s.connect(): + continue if s.family == socket.AF_INET: - name = '%s:%s (%s)' % ( s.ip, s.port, s.weight ) + name = '%s:%s (%s)' % (s.ip, s.port, s.weight) elif s.family == socket.AF_INET6: - name = '[%s]:%s (%s)' % ( s.ip, s.port, s.weight ) + name = '[%s]:%s (%s)' % (s.ip, s.port, s.weight) else: - name = 'unix:%s (%s)' % ( s.address, s.weight ) + name = 'unix:%s (%s)' % (s.address, s.weight) serverData = {} - data.append(( name, serverData )) + data.append((name, serverData)) s.send_cmd('stats items') readline = s.readline while 1: line = readline() - if not line or line.strip() == 'END': break + if not line or line.strip() == 'END': + break item = line.split(' ', 2) - #0 = STAT, 1 = ITEM, 2 = Value + # 0 = STAT, 1 = ITEM, 2 = Value slab = item[1].split(':', 2) - #0 = items, 1 = Slab #, 2 = Name + # 0 = items, 1 = Slab #, 2 = Name if slab[1] not in serverData: serverData[slab[1]] = {} serverData[slab[1]][slab[2]] = item[2] @@ -320,7 +327,8 @@ class Client(local): def flush_all(self): """Expire all data in memcache servers that are reachable.""" for s in self.servers: - if not s.connect(): continue + if not s.connect(): + continue s.flush() def debuglog(self, str): @@ -334,9 +342,7 @@ class Client(local): self.stats[func] += 1 def forget_dead_hosts(self): - """ - Reset every host in the pool to an "alive" state. - """ + """Reset every host in the pool to an "alive" state.""" for s in self.servers: s.deaduntil = 0 @@ -358,7 +364,7 @@ class Client(local): for i in range(Client._SERVER_RETRIES): server = self.buckets[serverhash % len(self.buckets)] if server.connect(): - #print "(using server %s)" % server, + # print("(using server %s)" % server,) return server, key serverhash = serverHashFunction(str(serverhash) + str(i)) return None, None @@ -368,54 +374,54 @@ class Client(local): s.close_socket() def delete_multi(self, keys, time=0, key_prefix=''): - ''' - Delete multiple keys in the memcache doing just one query. + """Delete multiple keys in the memcache doing just one query. - >>> notset_keys = mc.set_multi({'key1' : 'val1', 'key2' : 'val2'}) - >>> mc.get_multi(['key1', 'key2']) == {'key1' : 'val1', 'key2' : 'val2'} + >>> notset_keys = mc.set_multi({'a1' : 'val1', 'a2' : 'val2'}) + >>> mc.get_multi(['a1', 'a2']) == {'a1' : 'val1','a2' : 'val2'} 1 >>> mc.delete_multi(['key1', 'key2']) 1 >>> mc.get_multi(['key1', 'key2']) == {} 1 - - This method is recommended over iterated regular L{delete}s as it reduces total latency, since - your app doesn't have to wait for each round-trip of L{delete} before sending - the next one. + This method is recommended over iterated regular L{delete}s as + it reduces total latency, since your app doesn't have to wait + for each round-trip of L{delete} before sending the next one. @param keys: An iterable of keys to clear - @param time: number of seconds any subsequent set / update commands should fail. Defaults to 0 for no delay. - @param key_prefix: Optional string to prepend to each key when sending to memcache. - See docs for L{get_multi} and L{set_multi}. - + @param time: number of seconds any subsequent set / update + commands should fail. Defaults to 0 for no delay. + @param key_prefix: Optional string to prepend to each key when + sending to memcache. See docs for L{get_multi} and + L{set_multi}. @return: 1 if no failure in communication with any memcacheds. @rtype: int - - ''' + """ self._statlog('delete_multi') - server_keys, prefixed_to_orig_key = self._map_and_prefix_keys(keys, key_prefix) + server_keys, prefixed_to_orig_key = self._map_and_prefix_keys( + keys, key_prefix) # send out all requests on each server before reading anything dead_servers = [] rc = 1 - for server in server_keys.iterkeys(): + for server in six.iterkeys(server_keys): bigcmd = [] write = bigcmd.append - if time != None: - for key in server_keys[server]: # These are mangled keys - write("delete %s %d\r\n" % (key, time)) + if time is not None: + for key in server_keys[server]: # These are mangled keys + write("delete %s %d\r\n" % (key, time)) else: - for key in server_keys[server]: # These are mangled keys - write("delete %s\r\n" % key) + for key in server_keys[server]: # These are mangled keys + write("delete %s\r\n" % key) try: server.send_cmds(''.join(bigcmd)) - except socket.error, msg: + except socket.error as msg: rc = 0 - if isinstance(msg, tuple): msg = msg[1] + if isinstance(msg, tuple): + msg = msg[1] server.mark_dead(msg) dead_servers.append(server) @@ -427,8 +433,9 @@ class Client(local): try: for key in keys: server.expect("DELETED") - except socket.error, msg: - if isinstance(msg, tuple): msg = msg[1] + except socket.error as msg: + if isinstance(msg, tuple): + msg = msg[1] server.mark_dead(msg) rc = 0 return rc @@ -463,7 +470,7 @@ class Client(local): if not server: return 0 self._statlog(cmd) - if time != None and time != 0: + if time is not None and time != 0: cmd = "%s %s %d" % (cmd, key, time) else: cmd = "%s %s" % (cmd, key) @@ -471,23 +478,25 @@ class Client(local): try: server.send_cmd(cmd) line = server.readline() - if line and line.strip() in expected: return 1 + if line and line.strip() in expected: + return 1 self.debuglog('%s expected %s, got: %s' % (cmd, ' or '.join(expected), repr(line))) except socket.error, msg: - if isinstance(msg, tuple): msg = msg[1] + if isinstance(msg, tuple): + msg = msg[1] server.mark_dead(msg) return 0 def incr(self, key, delta=1): - """ - Sends a command to the server to atomically increment the value - for C{key} by C{delta}, or by 1 if C{delta} is unspecified. - Returns None if C{key} doesn't exist on server, otherwise it - returns the new value after incrementing. + """Sends a command to the server to atomically increment the + value for C{key} by C{delta}, or by 1 if C{delta} is + unspecified. Returns None if C{key} doesn't exist on server, + otherwise it returns the new value after incrementing. - Note that the value for C{key} must already exist in the memcache, - and it must be the string representation of an integer. + Note that the value for C{key} must already exist in the + memcache, and it must be the string representation of an + integer. >>> mc.set("counter", "20") # returns 1, indicating success 1 @@ -496,22 +505,25 @@ class Client(local): >>> mc.incr("counter") 22 - Overflow on server is not checked. Be aware of values approaching - 2**32. See L{decr}. + Overflow on server is not checked. Be aware of values + approaching 2**32. See L{decr}. + + @param delta: Integer amount to increment by (should be zero + or greater). - @param delta: Integer amount to increment by (should be zero or greater). @return: New value after incrementing. @rtype: int """ return self._incrdecr("incr", key, delta) def decr(self, key, delta=1): - """ - Like L{incr}, but decrements. Unlike L{incr}, underflow is checked and - new values are capped at 0. If server value is 1, a decrement of 2 - returns 0, not -1. + """Like L{incr}, but decrements. Unlike L{incr}, underflow is + checked and new values are capped at 0. If server value is 1, + a decrement of 2 returns 0, not -1. + + @param delta: Integer amount to decrement by (should be zero + or greater). - @param delta: Integer amount to decrement by (should be zero or greater). @return: New value after decrementing or None on error. @rtype: int """ @@ -528,18 +540,20 @@ class Client(local): try: server.send_cmd(cmd) line = server.readline() - if line == None or line.strip() =='NOT_FOUND': return None + if line is None or line.strip() == 'NOT_FOUND': + return None return int(line) - except socket.error, msg: - if isinstance(msg, tuple): msg = msg[1] + except socket.error as msg: + if isinstance(msg, tuple): + msg = msg[1] server.mark_dead(msg) return None - def add(self, key, val, time = 0, min_compress_len = 0): - ''' - Add new key with value. + def add(self, key, val, time=0, min_compress_len=0): + '''Add new key with value. - Like L{set}, but only stores in memcache if the key doesn't already exist. + Like L{set}, but only stores in memcache if the key doesn't + already exist. @return: Nonzero on success. @rtype: int @@ -583,67 +597,73 @@ class Client(local): '''Unconditionally sets a key to a given value in the memcache. The C{key} can optionally be an tuple, with the first element - being the server hash value and the second being the key. - If you want to avoid making this module calculate a hash value. - You may prefer, for example, to keep all of a given user's objects - on the same memcache server, so you could use the user's unique - id as the hash value. + being the server hash value and the second being the key. If + you want to avoid making this module calculate a hash value. + You may prefer, for example, to keep all of a given user's + objects on the same memcache server, so you could use the + user's unique id as the hash value. @return: Nonzero on success. @rtype: int - @param time: Tells memcached the time which this value should expire, either - as a delta number of seconds, or an absolute unix time-since-the-epoch - value. See the memcached protocol docs section "Storage Commands" - for more info on <exptime>. We default to 0 == cache forever. - @param min_compress_len: The threshold length to kick in auto-compression - of the value using the zlib.compress() routine. If the value being cached is - a string, then the length of the string is measured, else if the value is an - object, then the length of the pickle result is measured. If the resulting - attempt at compression yeilds a larger string than the input, then it is - discarded. For backwards compatability, this parameter defaults to 0, - indicating don't ever try to compress. + + @param time: Tells memcached the time which this value should + expire, either as a delta number of seconds, or an absolute + unix time-since-the-epoch value. See the memcached protocol + docs section "Storage Commands" for more info on <exptime>. We + default to 0 == cache forever. + + @param min_compress_len: The threshold length to kick in + auto-compression of the value using the zlib.compress() + routine. If the value being cached is a string, then the + length of the string is measured, else if the value is an + object, then the length of the pickle result is measured. If + the resulting attempt at compression yeilds a larger string + than the input, then it is discarded. For backwards + compatability, this parameter defaults to 0, indicating don't + ever try to compress. + ''' return self._set("set", key, val, time, min_compress_len) - def cas(self, key, val, time=0, min_compress_len=0): '''Sets a key to a given value in the memcache if it hasn't been altered since last fetched. (See L{gets}). The C{key} can optionally be an tuple, with the first element - being the server hash value and the second being the key. - If you want to avoid making this module calculate a hash value. - You may prefer, for example, to keep all of a given user's objects - on the same memcache server, so you could use the user's unique - id as the hash value. + being the server hash value and the second being the key. If + you want to avoid making this module calculate a hash value. + You may prefer, for example, to keep all of a given user's + objects on the same memcache server, so you could use the + user's unique id as the hash value. @return: Nonzero on success. @rtype: int - @param time: Tells memcached the time which this value should expire, - either as a delta number of seconds, or an absolute unix - time-since-the-epoch value. See the memcached protocol docs section - "Storage Commands" for more info on <exptime>. We default to - 0 == cache forever. + + @param time: Tells memcached the time which this value should + expire, either as a delta number of seconds, or an absolute + unix time-since-the-epoch value. See the memcached protocol + docs section "Storage Commands" for more info on <exptime>. We + default to 0 == cache forever. + @param min_compress_len: The threshold length to kick in - auto-compression of the value using the zlib.compress() routine. If - the value being cached is a string, then the length of the string is - measured, else if the value is an object, then the length of the - pickle result is measured. If the resulting attempt at compression - yeilds a larger string than the input, then it is discarded. For - backwards compatability, this parameter defaults to 0, indicating - don't ever try to compress. + auto-compression of the value using the zlib.compress() + routine. If the value being cached is a string, then the + length of the string is measured, else if the value is an + object, then the length of the pickle result is measured. If + the resulting attempt at compression yeilds a larger string + than the input, then it is discarded. For backwards + compatability, this parameter defaults to 0, indicating don't + ever try to compress. ''' return self._set("cas", key, val, time, min_compress_len) - def _map_and_prefix_keys(self, key_iterable, key_prefix): - """Compute the mapping of server (_Host instance) -> list of keys to stuff onto that server, as well as the mapping of - prefixed key -> original key. - - + """Compute the mapping of server (_Host instance) -> list of keys to + stuff onto that server, as well as the mapping of prefixed key + -> original key. """ # Check it just once ... - key_extra_len=len(key_prefix) + key_extra_len = len(key_prefix) if key_prefix and self.do_check_key: self.check_key(key_prefix) @@ -654,12 +674,18 @@ class Client(local): # build up a list for each server of all the keys we want. for orig_key in key_iterable: if isinstance(orig_key, tuple): - # Tuple of hashvalue, key ala _get_server(). Caller is essentially telling us what server to stuff this on. + # Tuple of hashvalue, key ala _get_server(). Caller is + # essentially telling us what server to stuff this on. # Ensure call to _get_server gets a Tuple as well. str_orig_key = str(orig_key[1]) - server, key = self._get_server((orig_key[0], key_prefix + str_orig_key)) # Gotta pre-mangle key before hashing to a server. Returns the mangled key. + + # Gotta pre-mangle key before hashing to a + # server. Returns the mangled key. + server, key = self._get_server( + (orig_key[0], key_prefix + str_orig_key)) else: - str_orig_key = str(orig_key) # set_multi supports int / long keys. + # set_multi supports int / long keys. + str_orig_key = str(orig_key) server, key = self._get_server(key_prefix + str_orig_key) # Now check to make sure key length is proper ... @@ -677,34 +703,39 @@ class Client(local): return (server_keys, prefixed_to_orig_key) def set_multi(self, mapping, time=0, key_prefix='', min_compress_len=0): - ''' - Sets multiple keys in the memcache doing just one query. + '''Sets multiple keys in the memcache doing just one query. >>> notset_keys = mc.set_multi({'key1' : 'val1', 'key2' : 'val2'}) - >>> mc.get_multi(['key1', 'key2']) == {'key1' : 'val1', 'key2' : 'val2'} + >>> mc.get_multi(['key1', 'key2']) == {'key1' : 'val1', + ... 'key2' : 'val2'} 1 - This method is recommended over regular L{set} as it lowers the - number of total packets flying around your network, reducing - total latency, since your app doesn't have to wait for each - round-trip of L{set} before sending the next one. + This method is recommended over regular L{set} as it lowers + the number of total packets flying around your network, + reducing total latency, since your app doesn't have to wait + for each round-trip of L{set} before sending the next one. @param mapping: A dict of key/value pairs to set. + @param time: Tells memcached the time which this value should - expire, either as a delta number of seconds, or an absolute - unix time-since-the-epoch value. See the memcached protocol - docs section "Storage Commands" for more info on <exptime>. We - default to 0 == cache forever. - @param key_prefix: Optional string to prepend to each key when + expire, either as a delta number of seconds, or an + absolute unix time-since-the-epoch value. See the + memcached protocol docs section "Storage Commands" for + more info on <exptime>. We default to 0 == cache forever. + + @param key_prefix: Optional string to prepend to each key when sending to memcache. Allows you to efficiently stuff these keys into a pseudo-namespace in memcache: >>> notset_keys = mc.set_multi( - ... {'key1' : 'val1', 'key2' : 'val2'}, key_prefix='subspace_') + ... {'key1' : 'val1', 'key2' : 'val2'}, + ... key_prefix='subspace_') >>> len(notset_keys) == 0 True - >>> mc.get_multi(['subspace_key1', 'subspace_key2']) == {'subspace_key1' : 'val1', 'subspace_key2' : 'val2'} + >>> mc.get_multi(['subspace_key1', + ... 'subspace_key2']) == {'subspace_key1': 'val1', + ... 'subspace_key2' : 'val2'} True Causes key 'subspace_key1' and 'subspace_key2' to be @@ -715,44 +746,49 @@ class Client(local): @param min_compress_len: The threshold length to kick in auto-compression of the value using the zlib.compress() - routine. If the value being cached is a string, then - the length of the string is measured, else if the value - is an object, then the length of the pickle result is - measured. If the resulting attempt at compression yeilds - a larger string than the input, then it is discarded. For + routine. If the value being cached is a string, then the + length of the string is measured, else if the value is an + object, then the length of the pickle result is + measured. If the resulting attempt at compression yeilds a + larger string than the input, then it is discarded. For backwards compatability, this parameter defaults to 0, indicating don't ever try to compress. - @return: List of keys which failed to be stored [ memcache out of - memory, etc. ]. - @rtype: list - ''' + @return: List of keys which failed to be stored [ memcache out + of memory, etc. ]. + @rtype: list + ''' self._statlog('set_multi') server_keys, prefixed_to_orig_key = self._map_and_prefix_keys( - mapping.iterkeys(), key_prefix) + six.iterkeys(mapping), key_prefix) # send out all requests on each server before reading anything dead_servers = [] - notstored = [] # original keys. + notstored = [] # original keys. - for server in server_keys.iterkeys(): + for server in six.iterkeys(server_keys): bigcmd = [] write = bigcmd.append try: - for key in server_keys[server]: # These are mangled keys + for key in server_keys[server]: # These are mangled keys store_info = self._val_to_store_info( - mapping[prefixed_to_orig_key[key]], - min_compress_len) + mapping[prefixed_to_orig_key[key]], + min_compress_len) if store_info: - write("set %s %d %d %d\r\n%s\r\n" % (key, store_info[0], - time, store_info[1], store_info[2])) + msg = "set %s %d %d %d\r\n%s\r\n" + write(msg % (key, + store_info[0], + time, + store_info[1], + store_info[2])) else: notstored.append(prefixed_to_orig_key[key]) server.send_cmds(''.join(bigcmd)) - except socket.error, msg: - if isinstance(msg, tuple): msg = msg[1] + except socket.error as msg: + if isinstance(msg, tuple): + msg = msg[1] server.mark_dead(msg) dead_servers.append(server) @@ -761,23 +797,28 @@ class Client(local): del server_keys[server] # short-circuit if there are no servers, just return all keys - if not server_keys: return(mapping.keys()) + if not server_keys: + return(mapping.keys()) - for server, keys in server_keys.iteritems(): + for server, keys in six.iteritems(server_keys): try: for key in keys: if server.readline() == 'STORED': continue else: - notstored.append(prefixed_to_orig_key[key]) #un-mangle. - except (_Error, socket.error), msg: - if isinstance(msg, tuple): msg = msg[1] + # un-mangle. + notstored.append(prefixed_to_orig_key[key]) + except (_Error, socket.error) as msg: + if isinstance(msg, tuple): + msg = msg[1] server.mark_dead(msg) return notstored def _val_to_store_info(self, val, min_compress_len): - """ - Transform val to a storable representation, returning a tuple of the flags, the length of the new value, and the new value itself. + """Transform val to a storable representation. + + Returns a tuple of the flags, the length of the new value, and + the new value itself. """ flags = 0 if isinstance(val, str): @@ -794,21 +835,23 @@ class Client(local): min_compress_len = 0 else: flags |= Client._FLAG_PICKLE - file = StringIO() + file = io.StringIO() if self.picklerIsKeyword: - pickler = self.pickler(file, protocol = self.pickleProtocol) + pickler = self.pickler(file, protocol=self.pickleProtocol) else: pickler = self.pickler(file, self.pickleProtocol) if self.persistent_id: pickler.persistent_id = self.persistent_id + print(repr(val)) pickler.dump(val) val = file.getvalue() lv = len(val) - # We should try to compress if min_compress_len > 0 and we could - # import zlib and this string is longer than our min threshold. - if min_compress_len and _supports_compress and lv > min_compress_len: - comp_val = compress(val) + # We should try to compress if min_compress_len > 0 and we + # could import zlib and this string is longer than our min + # threshold. + if min_compress_len and lv > min_compress_len: + comp_val = zlib.compress(val) # Only retain the result if the compression result is smaller # than the original. if len(comp_val) < lv: @@ -817,11 +860,12 @@ class Client(local): # silently do not store if value length exceeds maximum if self.server_max_value_length != 0 and \ - len(val) > self.server_max_value_length: return(0) + len(val) > self.server_max_value_length: + return(0) return (flags, len(val), val) - def _set(self, cmd, key, val, time, min_compress_len = 0): + def _set(self, cmd, key, val, time, min_compress_len=0): if self.do_check_key: self.check_key(key) server, key = self._get_server(key) @@ -832,24 +876,28 @@ class Client(local): self._statlog(cmd) store_info = self._val_to_store_info(val, min_compress_len) - if not store_info: return(0) + if not store_info: + return(0) if cmd == 'cas': if key not in self.cas_ids: return self._set('set', key, val, time, min_compress_len) fullcmd = "%s %s %d %d %d %d\r\n%s" % ( - cmd, key, store_info[0], time, store_info[1], - self.cas_ids[key], store_info[2]) + cmd, key, store_info[0], time, store_info[1], + self.cas_ids[key], store_info[2]) else: fullcmd = "%s %s %d %d %d\r\n%s" % ( - cmd, key, store_info[0], time, store_info[1], store_info[2]) + cmd, key, store_info[0], + time, store_info[1], store_info[2] + ) try: server.send_cmd(fullcmd) return(server.expect("STORED", raise_exception=True) - == "STORED") - except socket.error, msg: - if isinstance(msg, tuple): msg = msg[1] + == "STORED") + except socket.error as msg: + if isinstance(msg, tuple): + msg = msg[1] server.mark_dead(msg) return 0 @@ -860,7 +908,7 @@ class Client(local): try: if server._get_socket(): return _unsafe_set() - except (_ConnectionDeadError, socket.error), msg: + except (_ConnectionDeadError, socket.error) as msg: server.mark_dead(msg) return 0 @@ -879,13 +927,15 @@ class Client(local): rkey = flags = rlen = cas_id = None if cmd == 'gets': - rkey, flags, rlen, cas_id, = self._expect_cas_value(server, - raise_exception=True) + rkey, flags, rlen, cas_id, = self._expect_cas_value( + server, raise_exception=True + ) if rkey and self.cache_cas: self.cas_ids[rkey] = cas_id else: - rkey, flags, rlen, = self._expectvalue(server, - raise_exception=True) + rkey, flags, rlen, = self._expectvalue( + server, raise_exception=True + ) if not rkey: return None @@ -893,8 +943,9 @@ class Client(local): value = self._recv_value(server, flags, rlen) finally: server.expect("END", raise_exception=True) - except (_Error, socket.error), msg: - if isinstance(msg, tuple): msg = msg[1] + except (_Error, socket.error) as msg: + if isinstance(msg, tuple): + msg = msg[1] server.mark_dead(msg) return None @@ -908,7 +959,7 @@ class Client(local): if server.connect(): return _unsafe_get() return None - except (_ConnectionDeadError, socket.error), msg: + except (_ConnectionDeadError, socket.error) as msg: server.mark_dead(msg) return None @@ -927,54 +978,73 @@ class Client(local): return self._get('gets', key) def get_multi(self, keys, key_prefix=''): - ''' - Retrieves multiple keys from the memcache doing just one query. + '''Retrieves multiple keys from the memcache doing just one query. >>> success = mc.set("foo", "bar") >>> success = mc.set("baz", 42) - >>> mc.get_multi(["foo", "baz", "foobar"]) == {"foo": "bar", "baz": 42} + >>> mc.get_multi(["foo", "baz", "foobar"]) == { + ... "foo": "bar", "baz": 42 + ... } 1 >>> mc.set_multi({'k1' : 1, 'k2' : 2}, key_prefix='pfx_') == [] 1 - This looks up keys 'pfx_k1', 'pfx_k2', ... . Returned dict will just have unprefixed keys 'k1', 'k2'. - >>> mc.get_multi(['k1', 'k2', 'nonexist'], key_prefix='pfx_') == {'k1' : 1, 'k2' : 2} + This looks up keys 'pfx_k1', 'pfx_k2', ... . Returned dict + will just have unprefixed keys 'k1', 'k2'. + + >>> mc.get_multi(['k1', 'k2', 'nonexist'], + ... key_prefix='pfx_') == {'k1' : 1, 'k2' : 2} 1 - get_mult [ and L{set_multi} ] can take str()-ables like ints / longs as keys too. Such as your db pri key fields. - They're rotored through str() before being passed off to memcache, with or without the use of a key_prefix. - In this mode, the key_prefix could be a table name, and the key itself a db primary key number. + get_mult [ and L{set_multi} ] can take str()-ables like ints / + longs as keys too. Such as your db pri key fields. They're + rotored through str() before being passed off to memcache, + with or without the use of a key_prefix. In this mode, the + key_prefix could be a table name, and the key itself a db + primary key number. - >>> mc.set_multi({42: 'douglass adams', 46 : 'and 2 just ahead of me'}, key_prefix='numkeys_') == [] + >>> mc.set_multi({42: 'douglass adams', + ... 46: 'and 2 just ahead of me'}, + ... key_prefix='numkeys_') == [] 1 - >>> mc.get_multi([46, 42], key_prefix='numkeys_') == {42: 'douglass adams', 46 : 'and 2 just ahead of me'} + >>> mc.get_multi([46, 42], key_prefix='numkeys_') == { + ... 42: 'douglass adams', + ... 46: 'and 2 just ahead of me' + ... } 1 - This method is recommended over regular L{get} as it lowers the number of - total packets flying around your network, reducing total latency, since - your app doesn't have to wait for each round-trip of L{get} before sending - the next one. + This method is recommended over regular L{get} as it lowers + the number of total packets flying around your network, + reducing total latency, since your app doesn't have to wait + for each round-trip of L{get} before sending the next one. See also L{set_multi}. @param keys: An array of keys. - @param key_prefix: A string to prefix each key when we communicate with memcache. - Facilitates pseudo-namespaces within memcache. Returned dictionary keys will not have this prefix. - @return: A dictionary of key/value pairs that were available. If key_prefix was provided, the keys in the retured dictionary will not have it present. + @param key_prefix: A string to prefix each key when we + communicate with memcache. Facilitates pseudo-namespaces + within memcache. Returned dictionary keys will not have this + prefix. + + @return: A dictionary of key/value pairs that were + available. If key_prefix was provided, the keys in the retured + dictionary will not have it present. ''' self._statlog('get_multi') - server_keys, prefixed_to_orig_key = self._map_and_prefix_keys(keys, key_prefix) + server_keys, prefixed_to_orig_key = self._map_and_prefix_keys( + keys, key_prefix) # send out all requests on each server before reading anything dead_servers = [] - for server in server_keys.iterkeys(): + for server in six.iterkeys(server_keys): try: server.send_cmd("get %s" % " ".join(server_keys[server])) - except socket.error, msg: - if isinstance(msg, tuple): msg = msg[1] + except socket.error as msg: + if isinstance(msg, tuple): + msg = msg[1] server.mark_dead(msg) dead_servers.append(server) @@ -983,7 +1053,7 @@ class Client(local): del server_keys[server] retvals = {} - for server in server_keys.iterkeys(): + for server in six.iterkeys(server_keys): try: line = server.readline() while line and line != 'END': @@ -991,10 +1061,12 @@ class Client(local): # Bo Yang reports that this can sometimes be None if rkey is not None: val = self._recv_value(server, flags, rlen) - retvals[prefixed_to_orig_key[rkey]] = val # un-prefix returned key. + # un-prefix returned key. + retvals[prefixed_to_orig_key[rkey]] = val line = server.readline() - except (_Error, socket.error), msg: - if isinstance(msg, tuple): msg = msg[1] + except (_Error, socket.error) as msg: + if isinstance(msg, tuple): + msg = msg[1] server.mark_dead(msg) return retvals @@ -1021,19 +1093,19 @@ class Client(local): return (None, None, None) def _recv_value(self, server, flags, rlen): - rlen += 2 # include \r\n + rlen += 2 # include \r\n buf = server.recv(rlen) if len(buf) != rlen: raise _Error("received %d bytes when expecting %d" - % (len(buf), rlen)) + % (len(buf), rlen)) if len(buf) == rlen: buf = buf[:-2] # strip \r\n if flags & Client._FLAG_COMPRESSED: - buf = decompress(buf) + buf = zlib.decompress(buf) - if flags == 0 or flags == Client._FLAG_COMPRESSED: + if flags == 0 or flags == Client._FLAG_COMPRESSED: # Either a bare string or a compressed string now decompressed... val = buf elif flags & Client._FLAG_INTEGER: @@ -1042,12 +1114,12 @@ class Client(local): val = long(buf) elif flags & Client._FLAG_PICKLE: try: - file = StringIO(buf) + file = io.StringIO(buf) unpickler = self.unpickler(file) if self.persistent_load: unpickler.persistent_load = self.persistent_load val = unpickler.load() - except Exception, e: + except Exception as e: self.debuglog('Pickle error: %s\n' % e) return None else: @@ -1065,24 +1137,25 @@ class Client(local): Is not a string (Raises MemcachedKeyError) Is None (Raises MemcachedKeyError) """ - if isinstance(key, tuple): key = key[1] + if isinstance(key, tuple): + key = key[1] if not key: raise Client.MemcachedKeyNoneError("Key is None") - if isinstance(key, unicode): - raise Client.MemcachedStringEncodingError( - "Keys must be str()'s, not unicode. Convert your unicode " - "strings using mystring.encode(charset)!") - if not isinstance(key, str): + if isinstance(key, six.text_type): + key = key.encode('ascii') + if not isinstance(key, six.binary_type): raise Client.MemcachedKeyTypeError("Key must be str()'s") - if isinstance(key, basestring): + if isinstance(key, six.string_types): if self.server_max_key_length != 0 and \ - len(key) + key_extra_len > self.server_max_key_length: - raise Client.MemcachedKeyLengthError("Key length is > %s" - % self.server_max_key_length) + len(key) + key_extra_len > self.server_max_key_length: + raise Client.MemcachedKeyLengthError( + "Key length is > %s" % self.server_max_key_length + ) if not valid_key_chars_re.match(key): raise Client.MemcachedKeyCharacterError( - "Control characters not allowed") + "Control characters not allowed") + class _Host(object): @@ -1101,11 +1174,12 @@ class _Host(object): m = re.match(r'^(?P<proto>unix):(?P<path>.*)$', host) if not m: m = re.match(r'^(?P<proto>inet6):' - r'\[(?P<host>[^\[\]]+)\](:(?P<port>[0-9]+))?$', host) + r'\[(?P<host>[^\[\]]+)\](:(?P<port>[0-9]+))?$', host) if not m: m = re.match(r'^(?P<proto>inet):' - r'(?P<host>[^:]+)(:(?P<port>[0-9]+))?$', host) - if not m: m = re.match(r'^(?P<host>[^:]+)(:(?P<port>[0-9]+))?$', host) + r'(?P<host>[^:]+)(:(?P<port>[0-9]+))?$', host) + if not m: + m = re.match(r'^(?P<host>[^:]+)(:(?P<port>[0-9]+))?$', host) if not m: raise ValueError('Unable to parse connection string: "%s"' % host) @@ -1117,12 +1191,12 @@ class _Host(object): self.family = socket.AF_INET6 self.ip = hostData['host'] self.port = int(hostData.get('port') or 11211) - self.address = ( self.ip, self.port ) + self.address = (self.ip, self.port) else: self.family = socket.AF_INET self.ip = hostData['host'] self.port = int(hostData.get('port') or 11211) - self.address = ( self.ip, self.port ) + self.address = (self.ip, self.port) self.deaduntil = 0 self.socket = None @@ -1158,14 +1232,16 @@ class _Host(object): if self.socket: return self.socket s = socket.socket(self.family, socket.SOCK_STREAM) - if hasattr(s, 'settimeout'): s.settimeout(self.socket_timeout) + if hasattr(s, 'settimeout'): + s.settimeout(self.socket_timeout) try: s.connect(self.address) - except socket.timeout, msg: + except socket.timeout as msg: self.mark_dead("connect: %s" % msg) return None - except socket.error, msg: - if isinstance(msg, tuple): msg = msg[1] + except socket.error as msg: + if isinstance(msg, tuple): + msg = msg[1] self.mark_dead("connect: %s" % msg[1]) return None self.socket = s @@ -1184,13 +1260,14 @@ class _Host(object): self.socket.sendall(cmd + '\r\n') def send_cmds(self, cmds): - """ cmds already has trailing \r\n's applied """ + """cmds already has trailing \r\n's applied.""" self.socket.sendall(cmds) def readline(self, raise_exception=False): - """Read a line and return it. If "raise_exception" is set, - raise _ConnectionDeadError if the read fails, otherwise return - an empty string. + """Read a line and return it. + + If "raise_exception" is set, raise _ConnectionDeadError if the + read fails, otherwise return an empty string. """ buf = self.buffer if self.socket: @@ -1212,14 +1289,14 @@ class _Host(object): return '' buf += data - self.buffer = buf[index+2:] + self.buffer = buf[index + 2:] return buf[:index] def expect(self, text, raise_exception=False): line = self.readline(raise_exception) if line != text: self.debuglog("while expecting '%s', got unexpected response '%s'" - % (text, line)) + % (text, line)) return line def recv(self, rlen): @@ -1229,8 +1306,8 @@ class _Host(object): foo = self_socket_recv(max(rlen - len(buf), 4096)) buf += foo if not foo: - raise _Error( 'Read %d bytes, expecting %d, ' - 'read returned 0 length bytes' % ( len(buf), rlen )) + raise _Error('Read %d bytes, expecting %d, ' + 'read returned 0 length bytes' % (len(buf), rlen)) self.buffer = buf[rlen:] return buf[:rlen] @@ -1252,7 +1329,8 @@ class _Host(object): def _doctest(): - import doctest, memcache + import doctest + import memcache servers = ["127.0.0.1:11211"] mc = Client(servers, debug=1) globs = {"mc": mc} @@ -1260,9 +1338,9 @@ def _doctest(): if __name__ == "__main__": failures = 0 - print "Testing docstrings..." + print("Testing docstrings...") _doctest() - print "Running tests:" + print("Running tests:") print serverList = [["127.0.0.1:11211"]] if '--do-unix' in sys.argv: @@ -1275,24 +1353,28 @@ if __name__ == "__main__": if not isinstance(val, basestring): return "%s (%s)" % (val, type(val)) return "%s" % val + def test_setget(key, val): global failures - print "Testing set/get {'%s': %s} ..." % (to_s(key), to_s(val)), + print("Testing set/get {'%s': %s} ..." % (to_s(key), to_s(val)),) mc.set(key, val) newval = mc.get(key) if newval == val: - print "OK" + print("OK") return 1 else: - print "FAIL"; failures = failures + 1 + print("FAIL") + failures = failures + 1 return 0 - class FooStruct(object): + def __init__(self): self.bar = "baz" + def __str__(self): return "A FooStruct" + def __eq__(self, other): if isinstance(other, FooStruct): return self.bar == other.bar @@ -1300,139 +1382,141 @@ if __name__ == "__main__": test_setget("a_string", "some random string") test_setget("an_integer", 42) - if test_setget("long", long(1<<30)): - print "Testing delete ...", + if test_setget("long", long(1 << 30)): + print("Testing delete ...",) if mc.delete("long"): - print "OK" + print("OK") else: - print "FAIL"; failures = failures + 1 - print "Checking results of delete ..." - if mc.get("long") == None: - print "OK" + print("FAIL") + failures = failures + 1 + print("Checking results of delete ...") + if mc.get("long") is None: + print("OK") else: - print "FAIL"; failures = failures + 1 - print "Testing get_multi ...", - print mc.get_multi(["a_string", "an_integer"]) - - # removed from the protocol - #if test_setget("timed_delete", 'foo'): - # print "Testing timed delete ...", - # if mc.delete("timed_delete", 1): - # print "OK" - # else: - # print "FAIL"; failures = failures + 1 - # print "Checking results of timed delete ..." - # if mc.get("timed_delete") == None: - # print "OK" - # else: - # print "FAIL"; failures = failures + 1 - - print "Testing get(unknown value) ...", - print to_s(mc.get("unknown_value")) + print("FAIL") + failures = failures + 1 + print("Testing get_multi ...",) + print(mc.get_multi(["a_string", "an_integer"])) + print("Testing get(unknown value) ...",) + print(to_s(mc.get("unknown_value"))) f = FooStruct() test_setget("foostruct", f) - print "Testing incr ...", + print("Testing incr ...",) x = mc.incr("an_integer", 1) if x == 43: - print "OK" + print("OK") else: - print "FAIL"; failures = failures + 1 + print("FAIL") + failures = failures + 1 - print "Testing decr ...", + print("Testing decr ...",) x = mc.decr("an_integer", 1) if x == 42: - print "OK" + print("OK") else: - print "FAIL"; failures = failures + 1 + print("FAIL") + failures = failures + 1 sys.stdout.flush() # sanity tests - print "Testing sending spaces...", + print("Testing sending spaces...",) sys.stdout.flush() try: x = mc.set("this has spaces", 1) - except Client.MemcachedKeyCharacterError, msg: - print "OK" + except Client.MemcachedKeyCharacterError as msg: + print("OK") else: - print "FAIL"; failures = failures + 1 + print("FAIL") + failures = failures + 1 - print "Testing sending control characters...", + print("Testing sending control characters...",) try: x = mc.set("this\x10has\x11control characters\x02", 1) - except Client.MemcachedKeyCharacterError, msg: - print "OK" + except Client.MemcachedKeyCharacterError as msg: + print("OK") else: - print "FAIL"; failures = failures + 1 + print("FAIL") + failures = failures + 1 - print "Testing using insanely long key...", + print("Testing using insanely long key...",) try: - x = mc.set('a'*SERVER_MAX_KEY_LENGTH, 1) - except Client.MemcachedKeyLengthError, msg: - print "FAIL"; failures = failures + 1 + x = mc.set('a' * SERVER_MAX_KEY_LENGTH, 1) + except Client.MemcachedKeyLengthError as msg: + print("FAIL") + failures = failures + 1 else: - print "OK" + print("OK") try: - x = mc.set('a'*SERVER_MAX_KEY_LENGTH + 'a', 1) - except Client.MemcachedKeyLengthError, msg: - print "OK" + x = mc.set('a' * SERVER_MAX_KEY_LENGTH + 'a', 1) + except Client.MemcachedKeyLengthError as msg: + print("OK") else: - print "FAIL"; failures = failures + 1 + failures = failures + 1 + print("FAIL") - print "Testing sending a unicode-string key...", + print("Testing sending a unicode-string key...",) try: x = mc.set(unicode('keyhere'), 1) - except Client.MemcachedStringEncodingError, msg: - print "OK", + except Client.MemcachedStringEncodingError as msg: + print("OK",) else: - print "FAIL",; failures = failures + 1 + failures = failures + 1 + print("FAIL") try: - x = mc.set((unicode('a')*SERVER_MAX_KEY_LENGTH).encode('utf-8'), 1) - except: - print "FAIL",; failures = failures + 1 + x = mc.set((unicode('a') * SERVER_MAX_KEY_LENGTH) + .encode('utf-8'), 1) + except Exception: + failures = failures + 1 + print("FAIL") else: - print "OK", + print("OK",) import pickle s = pickle.loads('V\\u4f1a\np0\n.') try: - x = mc.set((s*SERVER_MAX_KEY_LENGTH).encode('utf-8'), 1) + x = mc.set((s * SERVER_MAX_KEY_LENGTH).encode('utf-8'), 1) except Client.MemcachedKeyLengthError: - print "OK" + print("OK") else: - print "FAIL"; failures = failures + 1 - - print "Testing using a value larger than the memcached value limit..." - print 'NOTE: "MemCached: while expecting[...]" is normal...' - x = mc.set('keyhere', 'a'*SERVER_MAX_VALUE_LENGTH) - if mc.get('keyhere') == None: - print "OK", + failures = failures + 1 + print("FAIL") + + print("Testing using a value larger than the memcached value limit...") + print('NOTE: "MemCached: while expecting[...]" is normal...') + x = mc.set('keyhere', 'a' * SERVER_MAX_VALUE_LENGTH) + if mc.get('keyhere') is None: + print("OK",) else: - print "FAIL",; failures = failures + 1 - x = mc.set('keyhere', 'a'*SERVER_MAX_VALUE_LENGTH + 'aaa') - if mc.get('keyhere') == None: - print "OK" + failures = failures + 1 + print("FAIL") + x = mc.set('keyhere', 'a' * SERVER_MAX_VALUE_LENGTH + 'aaa') + if mc.get('keyhere') is None: + print("OK") else: - print "FAIL"; failures = failures + 1 + failures = failures + 1 + print("FAIL") - print "Testing set_multi() with no memcacheds running", + print("Testing set_multi() with no memcacheds running",) mc.disconnect_all() - errors = mc.set_multi({'keyhere' : 'a', 'keythere' : 'b'}) + errors = mc.set_multi({'keyhere': 'a', 'keythere': 'b'}) if errors != []: - print "FAIL"; failures = failures + 1 + failures = failures + 1 + print("FAIL") else: - print "OK" + print("OK") - print "Testing delete_multi() with no memcacheds running", + print("Testing delete_multi() with no memcacheds running",) mc.disconnect_all() - ret = mc.delete_multi({'keyhere' : 'a', 'keythere' : 'b'}) + ret = mc.delete_multi({'keyhere': 'a', 'keythere': 'b'}) if ret != 1: - print "FAIL"; failures = failures + 1 + failures = failures + 1 + print("FAIL") else: - print "OK" + print("OK") if failures > 0: - print '*** THERE WERE FAILED TESTS' + print('*** THERE WERE FAILED TESTS') sys.exit(1) sys.exit(0) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..ffe2fce --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +six @@ -1,10 +1,10 @@ #!/usr/bin/env python -from setuptools import setup -import memcache +from setuptools import setup # noqa + setup(name="python-memcached", - version=memcache.__version__, + version="1.53", description="Pure python memcached client", long_description=open("README.md").read(), author="Evan Martin", @@ -14,13 +14,13 @@ setup(name="python-memcached", url="http://www.tummy.com/Community/software/python-memcached/", download_url="ftp://ftp.tummy.com/pub/python-memcached/", py_modules=["memcache"], + requirements=open('requirements.txt').read().split(), classifiers=[ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "License :: OSI Approved :: Python Software Foundation License", - "Operating System :: OS Independent", - "Programming Language :: Python", - "Topic :: Internet", - "Topic :: Software Development :: Libraries :: Python Modules", - ]) - + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "License :: OSI Approved :: Python Software Foundation License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Topic :: Internet", + "Topic :: Software Development :: Libraries :: Python Modules", + ]) diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..8f21390 --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,3 @@ +nose +coverage +hacking diff --git a/tests/Makefile b/tests/Makefile deleted file mode 100644 index a844aa4..0000000 --- a/tests/Makefile +++ /dev/null @@ -1,7 +0,0 @@ -TESTS = $(wildcard test_*.py) - -test: - @- $(foreach TEST,$(TESTS), \ - echo === Running test: $(TEST); \ - python $(TEST); \ - ) diff --git a/tests/test_setmulti.py b/tests/test_setmulti.py index 5947639..f10cb59 100644 --- a/tests/test_setmulti.py +++ b/tests/test_setmulti.py @@ -24,16 +24,16 @@ class test_Memcached_Set_Multi(unittest.TestCase): class FakeSocket(object): def __init__(self, *args): if DEBUG: - print 'FakeSocket{0!r}'.format(args) + print('FakeSocket{0!r}'.format(args)) self._recv_chunks = list(RECV_CHUNKS) def connect(self, *args): if DEBUG: - print 'FakeSocket.connect{0!r}'.format(args) + print('FakeSocket.connect{0!r}'.format(args)) def sendall(self, *args): if DEBUG: - print 'FakeSocket.sendall{0!r}'.format(args) + print('FakeSocket.sendall{0!r}'.format(args)) def recv(self, *args): if self._recv_chunks: @@ -41,12 +41,12 @@ class test_Memcached_Set_Multi(unittest.TestCase): else: data = '' if DEBUG: - print 'FakeSocket.recv{0!r} -> {1!r}'.format(args, data) + print('FakeSocket.recv{0!r} -> {1!r}'.format(args, data)) return data def close(self): if DEBUG: - print 'FakeSocket.close()' + print('FakeSocket.close()') self.old_socket = socket.socket socket.socket = FakeSocket @@ -62,7 +62,7 @@ class test_Memcached_Set_Multi(unittest.TestCase): self.assertEqual(sorted(bad_keys), ['bar', 'foo']) if DEBUG: - print 'set_multi({0!r}) -> {1!r}'.format(mapping, bad_keys) + print('set_multi({0!r}) -> {1!r}'.format(mapping, bad_keys)) if __name__ == '__main__': unittest.main() @@ -0,0 +1,24 @@ +[tox] +minversion = 1.6 +envlist = py26,py27,py33,pypy,pep8 +skipsdist = True + +[testenv] +usedevelop = True +# Customize pip command, add -U to force updates. +install_command = pip install -U {opts} {packages} +deps = -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt +commands = nosetests {posargs} + +[tox:jenkins] +downloadcache = ~/cache/pip + +[testenv:pep8] +commands = flake8 + +[testenv:cover] +commands = nosetests --with-coverage {posargs} + +[flake8] +exclude = .venv*,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*.egg,.update-venv |