From 3fde65eeae0019ec0971442886331fb7ffbcdffb Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 27 Jul 2015 18:08:10 +0200 Subject: More Python 3 fixes * Port set_multi() to Python 3 * Port delete_multi() to Python 3 * Fix _get_server() on Python 3 when the connection to the first server fails: encode to ASCII before calling serverHashFunction. * Fix expect(): don't decode line on Python 3, return the raw line * Add more unit tests * tox now also runs unit tests * Explicit the encoding when calling str.encode(): use 'utf-8' * test_memcache: close sockets in tearDown() * test_get_unknown_value(): delete the key to ensure that it doesn't exist --- memcache.py | 273 +++++++----------------------------------------------------- 1 file changed, 28 insertions(+), 245 deletions(-) (limited to 'memcache.py') diff --git a/memcache.py b/memcache.py index 60aa186..0cd78aa 100644 --- a/memcache.py +++ b/memcache.py @@ -258,12 +258,12 @@ class Client(threading.local): return key def _encode_cmd(self, cmd, key, headers, noreply, *args): - cmd_bytes = cmd.encode() if six.PY3 else cmd + cmd_bytes = cmd.encode('utf-8') if six.PY3 else cmd fullcmd = [cmd_bytes, b' ', key] if headers: if six.PY3: - headers = headers.encode() + headers = headers.encode('utf-8') fullcmd.append(b' ') fullcmd.append(headers) @@ -410,7 +410,10 @@ class Client(threading.local): if server.connect(): # print("(using server %s)" % server,) return server, key - serverhash = serverHashFunction(str(serverhash) + str(i)) + serverhash = str(serverhash) + str(i) + if isinstance(serverhash, six.text_type): + serverhash = serverhash.encode('ascii') + serverhash = serverHashFunction(serverhash) return None, None def disconnect_all(self): @@ -456,15 +459,15 @@ class Client(threading.local): for server in six.iterkeys(server_keys): bigcmd = [] write = bigcmd.append - extra = ' noreply' if noreply else '' if time is not None: - for key in server_keys[server]: # These are mangled keys - write("delete %s %d%s\r\n" % (key, time, extra)) + headers = str(time) else: - for key in server_keys[server]: # These are mangled keys - write("delete %s%s\r\n" % (key, extra)) + headers = None + for key in server_keys[server]: # These are mangled keys + cmd = self._encode_cmd('delete', key, headers, noreply, b'\r\n') + write(cmd) try: - server.send_cmds(''.join(bigcmd)) + server.send_cmds(b''.join(bigcmd)) except socket.error as msg: rc = 0 if isinstance(msg, tuple): @@ -483,7 +486,7 @@ class Client(threading.local): for server, keys in six.iteritems(server_keys): try: for key in keys: - server.expect("DELETED") + server.expect(b"DELETED") except socket.error as msg: if isinstance(msg, tuple): msg = msg[1] @@ -528,9 +531,10 @@ class Client(threading.local): return 0 self._statlog(cmd) if time is not None and time != 0: - fullcmd = self._encode_cmd(cmd, key, str(time), noreply) + headers = str(time) else: - fullcmd = self._encode_cmd(cmd, key, None, noreply) + headers = None + fullcmd = self._encode_cmd(cmd, key, headers, noreply) try: server.send_cmd(fullcmd) @@ -905,12 +909,12 @@ class Client(threading.local): # short-circuit if there are no servers, just return all keys if not server_keys: - return(mapping.keys()) + return list(mapping.keys()) for server, keys in six.iteritems(server_keys): try: for key in keys: - if server.readline() == 'STORED': + if server.readline() == b'STORED': continue else: # un-mangle. @@ -1039,7 +1043,7 @@ class Client(threading.local): self._statlog(cmd) try: - cmd_bytes = cmd.encode() if six.PY3 else cmd + cmd_bytes = cmd.encode('utf-8') if six.PY3 else cmd fullcmd = b''.join((cmd_bytes, b' ', key)) server.send_cmd(fullcmd) rkey = flags = rlen = cas_id = None @@ -1434,9 +1438,11 @@ class _Host(object): if self.debug and line != text: if six.PY3: text = text.decode('utf8') - line = line.decode('utf8', 'replace') + log_line = line.decode('utf8', 'replace') + else: + log_line = line self.debuglog("while expecting %r, got unexpected response %r" - % (text, line)) + % (text, log_line)) return line def recv(self, rlen): @@ -1472,236 +1478,13 @@ def _doctest(): import doctest import memcache servers = ["127.0.0.1:11211"] - mc = Client(servers, debug=1) + mc = memcache.Client(servers, debug=1) globs = {"mc": mc} - return doctest.testmod(memcache, globs=globs) - -if __name__ == "__main__": - failures = 0 - print("Testing docstrings...") - _doctest() - print("Running tests:") - print() - serverList = [["127.0.0.1:11211"]] - if '--do-unix' in sys.argv: - serverList.append([os.path.join(os.getcwd(), 'memcached.socket')]) - - for servers in serverList: - mc = Client(servers, debug=1) - - def to_s(val): - if not isinstance(val, _str_cls): - return "%s (%s)" % (val, type(val)) - return "%s" % val - - def test_setget(key, val, noreply=False): - global failures - print("Testing set/get (noreply=%s) {'%s': %s} ..." - % (noreply, to_s(key), to_s(val)), end=" ") - mc.set(key, val, noreply=noreply) - newval = mc.get(key) - if newval == val: - print("OK") - return 1 - else: - print("FAIL") - failures += 1 - return 0 - - class FooStruct(object): - - def __init__(self): - self.bar = "baz" - - def __str__(self): - return "A FooStruct" - - def __eq__(self, other): - if isinstance(other, FooStruct): - return self.bar == other.bar - return 0 - - test_setget("a_string", "some random string") - test_setget("a_string_2", "some random string", noreply=True) - test_setget("an_integer", 42) - test_setget("an_integer_2", 42, noreply=True) - if six.PY3: - ok = test_setget("long", 1 << 30) - else: - ok = test_setget("long", long(1 << 30)) - if ok: - print("Testing delete ...", end=" ") - if mc.delete("long"): - print("OK") - else: - print("FAIL") - failures += 1 - print("Checking results of delete ...", end=" ") - if mc.get("long") is None: - print("OK") - else: - print("FAIL") - failures += 1 - print("Testing get_multi ...",) - print(mc.get_multi(["a_string", "an_integer", "a_string_2", - "an_integer_2"])) - - # removed from the protocol - # if test_setget("timed_delete", 'foo'): - # print "Testing timed delete ...", - # if mc.delete("timed_delete", 1): - # print("OK") - # else: - # print("FAIL") - # failures += 1 - # print "Checking results of timed delete ..." - # if mc.get("timed_delete") is None: - # print("OK") - # else: - # print("FAIL") - # failures += 1 - - print("Testing get(unknown value) ...", end=" ") - print(to_s(mc.get("unknown_value"))) - - f = FooStruct() - test_setget("foostruct", f) - test_setget("foostruct_2", f, noreply=True) - - print("Testing incr ...", end=" ") - x = mc.incr("an_integer", 1) - if x == 43: - print("OK") - else: - print("FAIL") - failures += 1 - - print("Testing incr (noreply=True) ...", end=" ") - mc.incr("an_integer_2", 1, noreply=True) - x = mc.get("an_integer_2") - if x == 43: - print("OK") - else: - print("FAIL") - failures += 1 - - print("Testing decr ...", end=" ") - x = mc.decr("an_integer", 1) - if x == 42: - print("OK") - else: - print("FAIL") - failures += 1 - sys.stdout.flush() - - print("Testing decr (noreply=True) ...", end=" ") - mc.decr("an_integer_2", 1, noreply=True) - x = mc.get("an_integer_2") - if x == 42: - print("OK") - else: - print("FAIL") - failures += 1 - sys.stdout.flush() - - # sanity tests - print("Testing sending spaces...", end=" ") - sys.stdout.flush() - try: - x = mc.set("this has spaces", 1) - except Client.MemcachedKeyCharacterError as msg: - print("OK") - else: - print("FAIL") - failures += 1 - - print("Testing sending control characters...", end=" ") - try: - x = mc.set("this\x10has\x11control characters\x02", 1) - except Client.MemcachedKeyCharacterError as msg: - print("OK") - else: - print("FAIL") - failures += 1 - - print("Testing using insanely long key...", end=" ") - try: - x = mc.set('a'*SERVER_MAX_KEY_LENGTH, 1) - x = mc.set('a'*SERVER_MAX_KEY_LENGTH, 1, noreply=True) - except Client.MemcachedKeyLengthError as msg: - print("FAIL") - failures += 1 - else: - print("OK") - try: - x = mc.set('a'*SERVER_MAX_KEY_LENGTH + 'a', 1) - except Client.MemcachedKeyLengthError as msg: - print("OK") - else: - print("FAIL") - failures += 1 - - print("Testing sending a unicode-string key...", end=" ") - try: - x = mc.set(unicode('keyhere'), 1) - except Client.MemcachedStringEncodingError as msg: - print("OK", end=" ") - else: - print("FAIL", end=" ") - failures += 1 - try: - x = mc.set((unicode('a')*SERVER_MAX_KEY_LENGTH).encode('utf-8'), 1) - except Client.MemcachedKeyError: - print("FAIL", end=" ") - failures += 1 - else: - print("OK", end=" ") - s = pickle.loads('V\\u4f1a\np0\n.') - try: - x = mc.set((s * SERVER_MAX_KEY_LENGTH).encode('utf-8'), 1) - except Client.MemcachedKeyLengthError: - print("OK") - else: - print("FAIL") - failures += 1 - - print("Testing using a value larger than the memcached value limit...") - print('NOTE: "MemCached: while expecting[...]" is normal...') - x = mc.set('keyhere', 'a'*SERVER_MAX_VALUE_LENGTH) - if mc.get('keyhere') is None: - print("OK", end=" ") - else: - print("FAIL", end=" ") - failures += 1 - x = mc.set('keyhere', 'a'*SERVER_MAX_VALUE_LENGTH + 'aaa') - if mc.get('keyhere') is None: - print("OK") - else: - print("FAIL") - failures += 1 - - print("Testing set_multi() with no memcacheds running", end=" ") - mc.disconnect_all() - errors = mc.set_multi({'keyhere': 'a', 'keythere': 'b'}) - if errors != []: - print("FAIL") - failures += 1 - else: - print("OK") - - print("Testing delete_multi() with no memcacheds running", end=" ") - mc.disconnect_all() - ret = mc.delete_multi({'keyhere': 'a', 'keythere': 'b'}) - if ret != 1: - print("FAIL") - failures += 1 - else: - print("OK") - - if failures > 0: - print('*** THERE WERE FAILED TESTS') + results = doctest.testmod(memcache, globs=globs) + mc.disconnect_all() + print("Doctests: %s" % (results,)) + if results.failed: sys.exit(1) - sys.exit(0) # vim: ts=4 sw=4 et : -- cgit v1.2.1