diff options
author | Charles Gordon <charles@pinterest.com> | 2012-10-19 08:20:47 -0700 |
---|---|---|
committer | Charles Gordon <charles@pinterest.com> | 2012-10-19 08:20:47 -0700 |
commit | ace7918fefdee8a2c0d4b47806eec3cc7f1a57d2 (patch) | |
tree | beeafdc690e304fb6ad60d35ae51d99b76497131 | |
parent | e3ebaef32b81f7b2a2b2ed222eed7a3b7b806410 (diff) | |
download | pymemcache-ace7918fefdee8a2c0d4b47806eec3cc7f1a57d2.tar.gz |
Initial commit of all pymemcache files
-rw-r--r-- | .gitignore | 31 | ||||
-rw-r--r-- | pymemcache/__init__.py | 0 | ||||
-rw-r--r-- | pymemcache/client.py | 714 | ||||
-rw-r--r-- | pymemcache/test/__init__.py | 0 | ||||
-rw-r--r-- | pymemcache/test/benchmark.py | 82 | ||||
-rw-r--r-- | pymemcache/test/integration.py | 204 | ||||
-rw-r--r-- | pymemcache/test/test_client.py | 369 | ||||
-rw-r--r-- | setup.py | 11 |
8 files changed, 1411 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9c4f53e --- /dev/null +++ b/.gitignore @@ -0,0 +1,31 @@ +# compiled code. +*.py[co] + +# Packages +*.egg +*.egg-info +# dist +# build +eggs +# parts +# bin +# var +# sdist +develop-eggs +.installed.cfg + +# Installer logs +pip-log.txt + +# Unit test / coverage reports +.coverage +.tox + +#Translations +*.mo + +#Mr Developer +.mr.developer.cfg + +# Swap files. +*.swp diff --git a/pymemcache/__init__.py b/pymemcache/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/pymemcache/__init__.py diff --git a/pymemcache/client.py b/pymemcache/client.py new file mode 100644 index 0000000..ddb4871 --- /dev/null +++ b/pymemcache/client.py @@ -0,0 +1,714 @@ +""" +A comprehensive, fast, pure-Python memcached client library. + +Basic Usage: +------------ + + from pymemcache.client import Client + + client = Client(('localhost', 11211)) + client.set('some_key', 'some_value', noreply=True) + result = client.get('some_key') + + +Serialization: +-------------- + + import json + from pymemcache.client import Client + + def json_serializer(key, value): + if type(value) == str: + return value, 1 + return json.dumps(value), 2 + + def json_deserializer(key, value, flags): + if flags == 1: + return value + if flags == 2: + return json.loads(value) + raise Exception("Unknown serialization format") + + client = Client(('localhost', 11211), serializer=json_serializer, + deserializer=json_deserializer) + client.set('key', {'a':'b', 'c':'d'}, noreply=True) + result = client.get('key') + + +Best Practices: +--------------- + + - Always set the connect_timeout and timeout arguments in the constructor to + avoid blocking your process when memcached is slow. Consider setting them + to small values like 0.05 (50ms) or less. + - Use the "noreply" flag whenever possible for a significant performance + boost. + - Use get_many and gets_many whenever possible, as they result in less + round trip times for fetching multiple keys. + - Use the "ignore_exc" flag to treat memcache/network errors as cache misses + on calls to the get* methods. + + +Not Implemented: +---------------- + +The following features are not implemented by this library: + + - Retries: It generally isn't worth retrying failed memcached calls. Use the + ignore_exc flag to treat failures as cache misses. + - Pooling: coming soon? + - Clustering: coming soon? + - Key/value validation: it's relatively expensive to validate keys and values + on the client side, and memcached already does so on the server side. + - Unix sockets: coming soon? + - Binary protocol: coming soon? +""" + +__author__ = "Charles Gordon" + + +import socket + + +RECV_SIZE = 1024 +VALID_STORE_RESULTS = { + 'set': ('STORED',), + 'add': ('STORED', 'NOT_STORED'), + 'replace': ('STORED', 'NOT_STORED'), + 'append': ('STORED', 'NOT_STORED'), + 'prepend': ('STORED', 'NOT_STORED'), + 'cas': ('STORED', 'EXISTS', 'NOT_FOUND'), +} + + +class MemcacheError(Exception): + "Base exception class" + pass + + +class MemcacheUnknownCommandError(MemcacheError): + """Raised when memcached fails to parse a request, likely due to a bug in + this library or a version mismatch with memcached.""" + pass + + +class MemcacheClientError(MemcacheError): + """Raised when memcached fails to parse the arguments to a request, likely + due to a malformed key and/or value, a bug in this library, or a version + mismatch with memcached.""" + pass + + +class MemcacheServerError(MemcacheError): + """Raised when memcached reports a failure while processing a request, + likely due to a bug or transient issue in memcached.""" + pass + + +class MemcacheUnknownError(MemcacheError): + """Raised when this library receives a response from memcached that it + cannot parse, likely due to a bug in this library or a version mismatch + with memcached.""" + pass + + +class MemcacheUnexpectedCloseError(MemcacheError): + "Raised when the connection with memcached closes unexpectedly." + pass + + +class Client(object): + """ + A client for a single memcached server. + + Keys and Values: + ---------------- + + Keys must have a __str__() method which should return a str with no more + than 250 ASCII characters and no whitespace or control characters. Unicode + strings must be encoded (as UTF-8, for example) unless they consist only + of ASCII characters that are neither whitespace nor control characters. + + Values must have a __str__() method and a __len__() method (unless + serialization is being used, see below). The __str__() method can return + any str object, and the __len__() method must return the length of the + str returned. For instance, passing a list won't work, because the str + returned by list.__str__() is not the same length as the value returned + by list.__len__(). As with keys, unicode values must be encoded if they + contain characters not in the ASCII subset. + + Serialization and Deserialization: + ---------------------------------- + + The constructor takes two optional functions, one for "serialization" of + values, and one for "deserialization". The serialization function takes + two arguments, a key and a value, and returns a tuple of two elements, the + serialized value, and an integer in the range 0-65535 (the "flags"). The + deserialization function takes three parameters, a key, value and flags + and returns the deserialized value. + + Here is an example using JSON for non-str values: + + def serialize_json(key, value): + if type(value) == str: + return value, 1 + return json.dumps(value), 2 + + def deserialize_json(key, value, flags): + if flags == 1: + return value + if flags == 2: + return json.loads(value) + raise Exception("Unknown flags for value: {}".format(flags)) + + Error Handling: + --------------- + + All of the methods in this class that talk to memcached can throw one of + the following exceptions: + + * MemcacheUnknownCommandError + * MemcacheClientError + * MemcacheServerError + * MemcacheUnknownError + * MemcacheUnexpectedCloseError + * socket.timeout + * socket.error + + Instances of this class maintain a persistent connection to memcached + which is terminated when any of these exceptions are raised. The next + call to a method on the object will result in a new connection being made + to memcached. + """ + + def __init__(self, + server, + serializer=None, + deserializer=None, + connect_timeout=None, + timeout=None, + no_delay=False, + ignore_exc=False): + """ + Constructor. + + Args: + server: tuple(hostname, port) + serializer: optional function, see notes in the class docs. + deserializer: optional function, see notes in the class docs. + connect_timeout: optional float, seconds to wait for a connection to + the memcached server. Defaults to "forever" (uses the underlying + default socket timeout, which can be very long). + timeout: optional float, seconds to wait for send or recv calls on + the socket connected to memcached. Defaults to "forever" (uses the + underlying default socket timeout, which can be very long). + no_delay: optional bool, set the TCP_NODELAY flag, which may help + with performance in some cases. Defaults to False. + ignore_exc: optional bool, True to cause the "get", "gets", + "get_many" and "gets_many" calls to treat any errors as cache + misses. Defaults to False. + + Notes: + The constructor does not make a connection to memcached. The first + call to a method on the object will do that. + """ + self.server = server + self.serializer = serializer + self.deserializer = deserializer + self.connect_timeout = connect_timeout + self.timeout = timeout + self.no_delay = no_delay + self.ignore_exc = ignore_exc + self.sock = None + self.buf = '' + + def _connect(self): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(self.connect_timeout) + sock.connect(self.server) + sock.settimeout(self.timeout) + if self.no_delay: + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + self.sock = sock + + def close(self): + """Close the connetion to memcached, if it is open. The next call to a + method that requires a connection will re-open it.""" + if self.sock is not None: + try: + self.sock.close() + except Exception: + pass + self.sock = None + self.buf = '' + + def set(self, key, value, expire=0, noreply=False): + """ + The memcached "set" command. + + Args: + key: str, see class docs for details. + value: str, see class docs for details. + expire: optional int, number of seconds until the item is expired + from the cache, or zero for no expiry (the default). + noreply: optional bool, False to wait for the reply (the default). + + Returns: + The string 'STORED' on success, or raises an Exception on error (see + class documentation). + """ + return self._store_cmd('set', key, expire, noreply, value) + + def add(self, key, value, expire=0, noreply=False): + """ + The memcached "add" command. + + Args: + key: str, see class docs for details. + value: str, see class docs for details. + expire: optional int, number of seconds until the item is expired + from the cache, or zero for no expiry (the default). + noreply: optional bool, False to wait for the reply (the default). + + Returns: + The string 'STORED' if the value was stored, 'NOT_STORED' if the key + already existed, or an Exception on error (see class docs). + """ + return self._store_cmd('add', key, expire, noreply, value) + + def replace(self, key, value, expire=0, noreply=False): + """ + The memcached "replace" command. + + Args: + key: str, see class docs for details. + value: str, see class docs for details. + expire: optional int, number of seconds until the item is expired + from the cache, or zero for no expiry (the default). + noreply: optional bool, False to wait for the reply (the default). + + Returns: + The string 'STORED' if the value was stored, 'NOT_STORED' if the key + didn't already exist or an Exception on error (see class docs). + """ + return self._store_cmd('replace', key, expire, noreply, value) + + def append(self, key, value, expire=0, noreply=False): + """ + The memcached "append" command. + + Args: + key: str, see class docs for details. + value: str, see class docs for details. + expire: optional int, number of seconds until the item is expired + from the cache, or zero for no expiry (the default). + noreply: optional bool, False to wait for the reply (the default). + + Returns: + The string 'STORED' on success, or raises an Exception on error (see + the class docs). + """ + return self._store_cmd('append', key, expire, noreply, value) + + def prepend(self, key, value, expire=0, noreply=False): + """ + The memcached "prepend" command. + + Args: + key: str, see class docs for details. + value: str, see class docs for details. + expire: optional int, number of seconds until the item is expired + from the cache, or zero for no expiry (the default). + noreply: optional bool, False to wait for the reply (the default). + + Returns: + The string 'STORED' on success, or raises an Exception on error (see + the class docs). + """ + return self._store_cmd('prepend', key, expire, noreply, value) + + def cas(self, key, value, cas, expire=0, noreply=False): + """ + The memcached "cas" command. + + Args: + key: str, see class docs for details. + value: str, see class docs for details. + cas: int or str that only contains the characters '0'-'9'. + expire: optional int, number of seconds until the item is expired + from the cache, or zero for no expiry (the default). + noreply: optional bool, False to wait for the reply (the default). + + Returns: + The string 'STORED' if the value was stored, 'EXISTS' if the key + already existed with a different cas, 'NOT_FOUND' if the key didn't + exist or raises an Exception on error (see the class docs). + """ + return self._store_cmd('cas', key, expire, noreply, value, cas) + + def get(self, key): + """ + The memcached "get" command, but only for one key, as a convenience. + + Args: + key: str, see class docs for details. + + Returns: + The value for the key, or None if the key wasn't found, or raises + an Exception on error (see class docs). + """ + return self._fetch_cmd('get', [key], False).get(key, None) + + def get_many(self, keys): + """ + The memcached "get" command. + + Args: + keys: list(str), see class docs for details. + + Returns: + A dict in which the keys are elements of the "keys" argument list + and the values are values from the cache. The dict may contain all, + some or none of the given keys. An exception is raised on errors (see + the class docs for details). + """ + return self._fetch_cmd('get', keys, False) + + def gets(self, key): + """ + The memcached "gets" command for one key, as a convenience. + + Args: + key: str, see class docs for details. + + Returns: + A tuple of (key, cas), or (None, None) if the key was not found. + Raises an Exception on errors (see class docs for details). + """ + return self._fetch_cmd('gets', [key], True).get(key, (None, None)) + + def gets_many(self, keys): + """ + The memcached "gets" command. + + Args: + keys: list(str), see class docs for details. + + Returns: + A dict in which the keys are elements of the "keys" argument list and + the values are tuples of (value, cas) from the cache. The dict may + contain all, some or none of the given keys. An exception is raised + on errors (see the class docs for details). + """ + return self._fetch_cmd('gets', keys, True) + + def delete(self, key, noreply=False): + """ + The memcached "delete" command. + + Args: + key: str, see class docs for details. + + Returns: + The string 'DELTED' if the key existed, and was deleted, 'NOT_FOUND' + if the string did not exist, or raises an Exception on error (see the + class docs for details). + """ + cmd = 'delete {}{}\r\n'.format(key, ' noreply' if noreply else '') + return self._misc_cmd(cmd, 'delete', noreply) + + def incr(self, key, value, noreply=False): + """ + The memcached "incr" command. + + Args: + key: str, see class docs for details. + value: int, the amount by which to increment the value. + noreply: optional bool, False to wait for the reply (the default). + + Returns: + The string 'NOT_FOUND', or an integer which is the value of the key + after incrementing. Raises an Exception on errors (see the class docs + for details). + """ + cmd = "incr {} {}{}\r\n".format( + key, + str(value), + ' noreply' if noreply else '') + result = self._misc_cmd(cmd, 'incr', noreply) + if noreply: + return None + if result == 'NOT_FOUND': + return result + return int(result) + + def decr(self, key, value, noreply=False): + """ + The memcached "decr" command. + + Args: + key: str, see class docs for details. + value: int, the amount by which to increment the value. + noreply: optional bool, False to wait for the reply (the default). + + Returns: + The string 'NOT_FOUND', or an integer which is the value of the key + after decrementing. Raises an Exception on errors (see the class + docs for details). + """ + cmd = "decr {} {}{}\r\n".format( + key, + str(value), + ' noreply' if noreply else '') + result = self._misc_cmd(cmd, 'decr', noreply) + if noreply: + return None + if result == 'NOT_FOUND': + return result + return int(result) + + def touch(self, key, expire=0, noreply=False): + """ + The memcached "touch" command. + + Args: + key: str, see class docs for details. + expire: optional int, number of seconds until the item is expired + from the cache, or zero for no expiry (the default). + noreply: optional bool, False to wait for the reply (the default). + + Returns: + The string 'OK' if the value was stored or raises an Exception on + error (see the class docs). + """ + cmd = "touch {} {}{}\r\n".format( + key, + expire, + ' noreply' if noreply else '') + return self._misc_cmd(cmd, 'touch', noreply) + + def stats(self): + # TODO(charles) + pass + + def flush_all(self, delay=0, noreply=False): + """ + The memcached "flush_all" command. + + Args: + delay: optional int, the number of seconds to wait before flushing, + or zero to flush immediately (the default). + noreply: optional bool, False to wait for the response (the default). + + Returns: + The string 'OK' on success, or raises an Exception on error (see the + class docs). + """ + cmd = "flush_all {}{}\r\n".format(delay, ' noreply' if noreply else '') + return self._misc_cmd(cmd, 'flush_all', noreply) + + def quit(self): + """ + The memcached "quit" command. + + This will close the connection with memcached. Calling any other + method on this object will re-open the connection, so this object can + be re-used after quit. + """ + cmd = "quit\r\n" + self._misc_cmd(cmd, 'quit', True) + self.close() + + def _raise_errors(self, line, name): + if line.startswith('ERROR'): + raise MemcacheUnknownCommandError(name) + + if line.startswith('CLIENT_ERROR'): + error = line[line.find(' ') + 1:] + raise MemcacheClientError(error) + + if line.startswith('SERVER_ERROR'): + error = line[line.find(' ') + 1:] + raise MemcacheServerError(error) + + def _fetch_cmd(self, name, keys, expect_cas): + if not self.sock: + self._connect() + + cmd = '{} {}\r\n'.format(name, ' '.join(keys)) + try: + self.sock.sendall(cmd) + + result = {} + while True: + self.buf, line = _readline(self.sock, self.buf) + self._raise_errors(line, name) + + if line == 'END': + return result + elif line.startswith('VALUE'): + if expect_cas: + _, key, flags, size, cas = line.split() + else: + _, key, flags, size = line.split() + + self.buf, value = _readvalue(self.sock, + self.buf, + int(size)) + + if self.deserializer: + value = self.deserializer(value, int(flags)) + + if expect_cas: + result[key] = (value, cas) + else: + result[key] = value + else: + raise MemcacheUnknownError(line[:32]) + except Exception: + self.close() + if self.ignore_exc: + return {} + raise + + def _store_cmd(self, name, key, expire, noreply, data, cas=None): + if not self.sock: + self._connect() + + if self.serializer: + data, flags = self.serializer(data) + else: + flags = 0 + + if cas is not None and noreply: + extra = ' {} noreply'.format(cas) + elif cas is not None and not noreply: + extra = ' {}'.format(cas) + elif cas is None and noreply: + extra = ' noreply' + else: + extra = '' + + cmd = '{} {} {} {} {}{}\r\n{}\r\n'.format( + name, + key, + flags, + expire, + len(data), + extra, + data) + + try: + self.sock.sendall(cmd) + + if noreply: + return + + self.buf, line = _readline(self.sock, self.buf) + self._raise_errors(line, name) + + if line in VALID_STORE_RESULTS[name]: + return line + else: + raise MemcacheUnknownError(line[:32]) + except Exception: + self.close() + raise + + def _misc_cmd(self, cmd, cmd_name, noreply): + if not self.sock: + self._connect() + + try: + self.sock.sendall(cmd) + + if noreply: + return + + _, line = _readline(self.sock, '') + self._raise_errors(line, cmd_name) + + return line + except Exception: + self.close() + raise + + +def _readline(sock, buf): + """Read line of text from the socket. + + Read a line of text (delimited by "\r\n") from the socket, and + return that line along with any trailing characters read from the + socket. + + Args: + sock: Socket object, should be connected. + buf: String, zero or more characters, returned from an earlier + call to _readline or _readvalue (pass an empty string on the + first call). + + Returns: + A tuple of (buf, line) where line is the full line read from the + socket (minus the "\r\n" characters) and buf is any trailing + characters read after the "\r\n" was found (which may be an empty + string). + + """ + chunks = [] + last_char = '' + + while True: + idx = buf.find('\r\n') + # We're reading in chunks, so "\r\n" could appear in one chunk, + # or across the boundary of two chunks, so we check for both + # cases. + if idx != -1: + before, sep, after = buf.partition("\r\n") + chunks.append(before) + return after, ''.join(chunks) + elif last_char == '\r' and buf[0] == '\n': + # Strip the last character from the last chunk. + chunks[-1] = chunks[-1][:-1] + return buf[1:], ''.join(chunks) + + if buf: + chunks.append(buf) + last_char = buf[-1] + + buf = sock.recv(RECV_SIZE) + if not buf: + raise MemcacheUnexpectedCloseError() + + +def _readvalue(sock, buf, size): + """Read specified amount of bytes from the socket. + + Read size bytes, followed by the "\r\n" characters, from the socket, + and return those bytes and any trailing bytes read after the "\r\n". + + Args: + sock: Socket object, should be connected. + buf: String, zero or more characters, returned from an earlier + call to _readline or _readvalue (pass an empty string on the + first call). + size: Integer, number of bytes to read from the socket. + + Returns: + A tuple of (buf, value) where value is the bytes read from the + socket (there will be exactly size bytes) and buf is trailing + characters read after the "\r\n" following the bytes (but not + including the \r\n). + + """ + chunks = [] + rlen = size + 2 + while rlen - len(buf) > 0: + if buf: + rlen -= len(buf) + chunks.append(buf) + buf = sock.recv(RECV_SIZE) + if not buf: + raise MemcacheUnexpectedCloseError() + + chunks.append(buf[:rlen - 2]) + return buf[rlen:], ''.join(chunks) diff --git a/pymemcache/test/__init__.py b/pymemcache/test/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/pymemcache/test/__init__.py diff --git a/pymemcache/test/benchmark.py b/pymemcache/test/benchmark.py new file mode 100644 index 0000000..28f60c3 --- /dev/null +++ b/pymemcache/test/benchmark.py @@ -0,0 +1,82 @@ +import argparse +import time + + +def test_client(name, client, size, count): + client.flush_all() + + value = 'X' * size + + start = time.time() + + for i in xrange(count): + client.set(str(i), value) + + for i in xrange(count): + client.get(str(i)) + + duration = time.time() - start + print "{}: {}".format(name, duration) + + +def test_pylibmc(host, port, size, count): + try: + import pylibmc + except Exception: + print "Could not import pylibmc, skipping test..." + return + + client = pylibmc.Client(['{}:{}'.format(host, port)]) + client.behaviors = {"tcp_nodelay": True} + test_client('pylibmc', client, size, count) + + +def test_memcache(host, port, size, count): + try: + import memcache + except Exception: + print "Could not import pymemcache.client, skipping test..." + return + + client = memcache.Client(['{}:{}'.format(host, port)]) + test_client('memcache', client, size, count) + + +def test_pymemcache(host, port, size, count): + try: + import pymemcache.client + except Exception: + print "Could not import pymemcache.client, skipping test..." + return + + client = pymemcache.client.Client((host, port)) + test_client('pymemcache', client, size, count) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('-s', '--server', + metavar='HOST', + required=True) + parser.add_argument('-p', '--port', + metavar='PORT', + type=int, + required=True) + parser.add_argument('-z', '--size', + metavar='SIZE', + default=1024, + type=int) + parser.add_argument('-c', '--count', + metavar='COUNT', + default=10000, + type=int) + + args = parser.parse_args() + + test_pylibmc(args.server, args.port, args.size, args.count) + test_memcache(args.server, args.port, args.size, args.count) + test_pymemcache(args.server, args.port, args.size, args.count) + + +if __name__ == '__main__': + main() diff --git a/pymemcache/test/integration.py b/pymemcache/test/integration.py new file mode 100644 index 0000000..26f620e --- /dev/null +++ b/pymemcache/test/integration.py @@ -0,0 +1,204 @@ +import argparse +import json + +from pymemcache.client import Client, MemcacheClientError +from nose import tools + + +def get_set_test(host, port): + client = Client((host, port)) + client.flush_all() + + result = client.get('key') + tools.assert_equal(result, None) + + client.set('key', 'value') + result = client.get('key') + tools.assert_equal(result, 'value') + + client.set('key2', 'value2', noreply=True) + result = client.get('key2') + tools.assert_equal(result, 'value2') + + result = client.get_many(['key', 'key2']) + tools.assert_equal(result, {'key': 'value', 'key2': 'value2'}) + + +def add_replace_test(host, port): + client = Client((host, port)) + client.flush_all() + + result = client.add('key', 'value') + tools.assert_equal(result, 'STORED') + result = client.get('key') + tools.assert_equal(result, 'value') + + result = client.add('key', 'value2') + tools.assert_equal(result, 'NOT_STORED') + result = client.get('key') + tools.assert_equal(result, 'value') + + result = client.replace('key1', 'value1') + tools.assert_equal(result, 'NOT_STORED') + result = client.get('key1') + tools.assert_equal(result, None) + + result = client.replace('key', 'value2') + tools.assert_equal(result, 'STORED') + result = client.get('key') + tools.assert_equal(result, 'value2') + + +def append_prepend_test(host, port): + client = Client((host, port)) + client.flush_all() + + result = client.append('key', 'value') + tools.assert_equal(result, 'NOT_STORED') + result = client.get('key') + tools.assert_equal(result, None) + + result = client.set('key', 'value') + tools.assert_equal(result, 'STORED') + result = client.append('key', 'after') + tools.assert_equal(result, 'STORED') + result = client.get('key') + tools.assert_equal(result, 'valueafter') + + result = client.prepend('key1', 'value') + tools.assert_equal(result, 'NOT_STORED') + result = client.get('key1') + tools.assert_equal(result, None) + + result = client.prepend('key', 'before') + tools.assert_equal(result, 'STORED') + result = client.get('key') + tools.assert_equal(result, 'beforevalueafter') + + +def cas_test(host, port): + client = Client((host, port)) + client.flush_all() + + result = client.cas('key', 'value', '1') + tools.assert_equal(result, 'NOT_FOUND') + + result = client.set('key', 'value') + tools.assert_equal(result, 'STORED') + + result = client.cas('key', 'value', '1') + tools.assert_equal(result, 'EXISTS') + + result, cas = client.gets('key') + tools.assert_equal(result, 'value') + + result = client.cas('key', 'value1', cas) + tools.assert_equal(result, 'STORED') + + result = client.cas('key', 'value2', cas) + tools.assert_equal(result, 'EXISTS') + + +def gets_test(host, port): + client = Client((host, port)) + client.flush_all() + + result = client.gets('key') + tools.assert_equal(result, (None, None)) + + result = client.set('key', 'value') + tools.assert_equal(result, 'STORED') + result = client.gets('key') + tools.assert_equal(result[0], 'value') + + +def delete_test(host, port): + client = Client((host, port)) + client.flush_all() + + result = client.delete('key') + tools.assert_equal(result, 'NOT_FOUND') + + result = client.get('key') + tools.assert_equal(result, None) + result = client.set('key', 'value') + tools.assert_equal(result, 'STORED') + result = client.delete('key') + tools.assert_equal(result, 'DELETED') + result = client.get('key') + tools.assert_equal(result, None) + + +def incr_decr_test(host, port): + client = Client((host, port)) + client.flush_all() + + result = client.incr('key', 1) + tools.assert_equal(result, 'NOT_FOUND') + + result = client.set('key', '0') + tools.assert_equal(result, 'STORED') + result = client.incr('key', 1) + tools.assert_equal(result, 1) + + def _bad_int(): + client.incr('key', 'foobar') + + tools.assert_raises(MemcacheClientError, _bad_int) + + result = client.decr('key1', 1) + tools.assert_equal(result, 'NOT_FOUND') + + result = client.decr('key', 1) + tools.assert_equal(result, 0) + result = client.get('key') + tools.assert_equal(result, '0') + + +def misc_test(host, port): + client = Client((host, port)) + client.flush_all() + + +def test_serialization_deserialization(host, port): + def _ser(value): + return json.dumps(value), 1 + + def _des(value, flags): + if flags == 1: + return json.loads(value) + return value + + client = Client((host, port), serializer=_ser, deserializer=_des) + client.flush_all() + + value = {'a': 'b', 'c': ['d']} + client.set('key', value) + result = client.get('key') + tools.assert_equal(result, value) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('-s', '--server', + metavar='HOST', + required=True) + parser.add_argument('-p', '--port', + metavar='PORT', + type=int, + required=True) + + args = parser.parse_args() + + get_set_test(args.server, args.port) + add_replace_test(args.server, args.port) + append_prepend_test(args.server, args.port) + cas_test(args.server, args.port) + gets_test(args.server, args.port) + delete_test(args.server, args.port) + incr_decr_test(args.server, args.port) + misc_test(args.server, args.port) + test_serialization_deserialization(args.server, args.port) + +if __name__ == '__main__': + main() diff --git a/pymemcache/test/test_client.py b/pymemcache/test/test_client.py new file mode 100644 index 0000000..26e2be6 --- /dev/null +++ b/pymemcache/test/test_client.py @@ -0,0 +1,369 @@ +import collections +import json + +from nose import tools +from pymemcache.client import Client, MemcacheUnknownCommandError +from pymemcache.client import MemcacheClientError, MemcacheServerError +from pymemcache.client import MemcacheUnknownError + + +class MockSocket(object): + def __init__(self, recv_bufs): + self.recv_bufs = collections.deque(recv_bufs) + self.send_bufs = [] + self.closed = False + + def sendall(self, value): + self.send_bufs.append(value) + + def close(self): + self.closed = True + + def recv(self, size): + value = self.recv_bufs.popleft() + if isinstance(value, Exception): + raise value + return value + + +def test_set_success(): + client = Client(None) + client.sock = MockSocket(['STORED\r\n']) + result = client.set('key', 'value') + tools.assert_equal(result, 'STORED') + tools.assert_equal(client.sock.closed, False) + tools.assert_equal(len(client.sock.send_bufs), 1) + + +def test_set_error(): + client = Client(None) + client.sock = MockSocket(['ERROR\r\n']) + + def _set(): + client.set('key', 'value') + + tools.assert_raises(MemcacheUnknownCommandError, _set) + + +def test_set_client_error(): + client = Client(None) + client.sock = MockSocket(['CLIENT_ERROR some message\r\n']) + + def _set(): + client.set('key', 'value') + + tools.assert_raises(MemcacheClientError, _set) + + +def test_set_server_error(): + client = Client(None) + client.sock = MockSocket(['SERVER_ERROR some message\r\n']) + + def _set(): + client.set('key', 'value') + + tools.assert_raises(MemcacheServerError, _set) + + +def test_set_unknown_error(): + client = Client(None) + client.sock = MockSocket(['foobarbaz\r\n']) + + def _set(): + client.set('key', 'value') + + tools.assert_raises(MemcacheUnknownError, _set) + + +def test_set_noreply(): + client = Client(None) + client.sock = MockSocket([]) + result = client.set('key', 'value', noreply=True) + tools.assert_equal(result, None) + + +def test_set_exception(): + client = Client(None) + client.sock = MockSocket([Exception('fail')]) + + def _set(): + client.set('key', 'value') + + tools.assert_raises(Exception, _set) + tools.assert_equal(client.sock, None) + tools.assert_equal(client.buf, '') + + +def test_add_stored(): + client = Client(None) + client.sock = MockSocket(['STORED\r', '\n']) + result = client.add('key', 'value') + tools.assert_equal(result, 'STORED') + + +def test_add_not_stored(): + client = Client(None) + client.sock = MockSocket(['NOT_', 'STOR', 'ED', '\r\n']) + result = client.add('key', 'value') + tools.assert_equal(result, 'NOT_STORED') + + +def test_replace_stored(): + client = Client(None) + client.sock = MockSocket(['STORED\r\n']) + result = client.replace('key', 'value') + tools.assert_equal(result, 'STORED') + + +def test_replace_not_stored(): + client = Client(None) + client.sock = MockSocket(['NOT_STORED\r\n']) + result = client.replace('key', 'value') + tools.assert_equal(result, 'NOT_STORED') + + +def test_append_stored(): + client = Client(None) + client.sock = MockSocket(['STORED\r\n']) + result = client.append('key', 'value') + tools.assert_equal(result, 'STORED') + + +def test_prepend_stored(): + client = Client(None) + client.sock = MockSocket(['STORED\r\n']) + result = client.prepend('key', 'value') + tools.assert_equal(result, 'STORED') + + +def test_cas_stored(): + client = Client(None) + client.sock = MockSocket(['STORED\r\n']) + result = client.cas('key', 'value', 'cas') + tools.assert_equal(result, 'STORED') + + +def test_cas_exists(): + client = Client(None) + client.sock = MockSocket(['EXISTS\r\n']) + result = client.cas('key', 'value', 'cas') + tools.assert_equal(result, 'EXISTS') + + +def test_cas_not_found(): + client = Client(None) + client.sock = MockSocket(['NOT_FOUND\r\n']) + result = client.cas('key', 'value', 'cas') + tools.assert_equal(result, 'NOT_FOUND') + + +def test_get_not_found(): + client = Client(None) + client.sock = MockSocket(['END\r\n']) + result = client.get('key') + tools.assert_equal(result, None) + + +def test_get_found(): + client = Client(None) + client.sock = MockSocket(['VALUE key 0 5\r\nvalue\r\nEND\r\n']) + result = client.get('key') + tools.assert_equal(result, 'value') + + +def test_get_error(): + client = Client(None) + client.sock = MockSocket(['ERROR\r\n']) + + def _get(): + client.get('key') + + tools.assert_raises(MemcacheUnknownCommandError, _get) + + +def test_get_many_none_found(): + client = Client(None) + client.sock = MockSocket(['END\r\n']) + result = client.get_many(['key1', 'key2']) + tools.assert_equal(result, {}) + + +def test_get_many_some_found(): + client = Client(None) + client.sock = MockSocket(['VALUE key1 0 6\r\nvalue1\r\nEND\r\n']) + result = client.get_many(['key1', 'key2']) + tools.assert_equal(result, {'key1': 'value1'}) + + +def test_get_many_all_found(): + client = Client(None) + client.sock = MockSocket(['VALUE key1 0 6\r\nvalue1\r\n' + 'VALUE key2 0 6\r\nvalue2\r\nEND\r\n']) + result = client.get_many(['key1', 'key2']) + tools.assert_equal(result, {'key1': 'value1', 'key2': 'value2'}) + + +def test_get_unknown_error(): + client = Client(None) + client.sock = MockSocket(['foobarbaz\r\n']) + + def _get(): + client.get('key') + + tools.assert_raises(MemcacheUnknownError, _get) + + +def test_gets_not_found(): + client = Client(None) + client.sock = MockSocket(['END\r\n']) + result = client.gets('key') + tools.assert_equal(result, (None, None)) + + +def test_gets_found(): + client = Client(None) + client.sock = MockSocket(['VALUE key 0 5 10\r\nvalue\r\nEND\r\n']) + result = client.gets('key') + tools.assert_equal(result, ('value', '10')) + + +def test_gets_many_none_found(): + client = Client(None) + client.sock = MockSocket(['END\r\n']) + result = client.gets_many(['key1', 'key2']) + tools.assert_equal(result, {}) + + +def test_gets_many_some_found(): + client = Client(None) + client.sock = MockSocket(['VALUE key1 0 6 11\r\nvalue1\r\nEND\r\n']) + result = client.gets_many(['key1', 'key2']) + tools.assert_equal(result, {'key1': ('value1', '11')}) + + +def test_get_recv_chunks(): + client = Client(None) + client.sock = MockSocket(['VALUE key', ' 0 5\r', '\nvalue', '\r\n', + 'END', '\r', '\n']) + result = client.get('key') + tools.assert_equal(result, 'value') + + +def test_delete_not_found(): + client = Client(None) + client.sock = MockSocket(['NOT_FOUND\r\n']) + result = client.delete('key') + tools.assert_equal(result, 'NOT_FOUND') + + +def test_delete_found(): + client = Client(None) + client.sock = MockSocket(['DELETED\r\n']) + result = client.delete('key') + tools.assert_equal(result, 'DELETED') + + +def test_delete_noreply(): + client = Client(None) + client.sock = MockSocket([]) + result = client.delete('key', noreply=True) + tools.assert_equal(result, None) + + +def test_delete_exception(): + client = Client(None) + client.sock = MockSocket([Exception('fail')]) + + def _delete(): + client.delete('key') + + tools.assert_raises(Exception, _delete) + tools.assert_equal(client.sock, None) + tools.assert_equal(client.buf, '') + + +def test_incr_not_found(): + client = Client(None) + client.sock = MockSocket(['NOT_FOUND\r\n']) + result = client.incr('key', 1) + tools.assert_equal(result, 'NOT_FOUND') + + +def test_incr_found(): + client = Client(None) + client.sock = MockSocket(['1\r\n']) + result = client.incr('key', 1) + tools.assert_equal(result, 1) + + +def test_incr_noreply(): + client = Client(None) + client.sock = MockSocket([]) + result = client.incr('key', 1, noreply=True) + tools.assert_equal(result, None) + + +def test_incr_exception(): + client = Client(None) + client.sock = MockSocket([Exception('fail')]) + + def _incr(): + client.incr('key', 1) + + tools.assert_raises(Exception, _incr) + tools.assert_equal(client.sock, None) + tools.assert_equal(client.buf, '') + + +def test_decr_not_found(): + client = Client(None) + client.sock = MockSocket(['NOT_FOUND\r\n']) + result = client.decr('key', 1) + tools.assert_equal(result, 'NOT_FOUND') + + +def test_decr_found(): + client = Client(None) + client.sock = MockSocket(['1\r\n']) + result = client.decr('key', 1) + tools.assert_equal(result, 1) + + +def test_flush_all(): + client = Client(None) + client.sock = MockSocket(['OK\r\n']) + result = client.flush_all() + tools.assert_equal(result, 'OK') + + +def test_touch_not_found(): + client = Client(None) + client.sock = MockSocket(['NOT_FOUND\r\n']) + result = client.touch('key') + tools.assert_equal(result, 'NOT_FOUND') + + +def test_touch_found(): + client = Client(None) + client.sock = MockSocket(['TOUCHED\r\n']) + result = client.touch('key') + tools.assert_equal(result, 'TOUCHED') + + +def test_quit(): + client = Client(None) + client.sock = MockSocket([]) + result = client.quit() + tools.assert_equal(result, None) + tools.assert_equal(client.sock, None) + tools.assert_equal(client.buf, '') + + +def test_serialization(): + def _ser(value): + return json.dumps(value), 0 + + client = Client(None, serializer=_ser) + client.sock = MockSocket(['STORED\r\n']) + client.set('key', {'a': 'b', 'c': 'd'}) + print client.sock.send_bufs diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..fd0d533 --- /dev/null +++ b/setup.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python + +from setuptools import setup, find_packages + +setup( + name = 'pymemcache', + version = '0.1', + packages = find_packages(), + setup_requires = ['nose>=1.0'], +) + |