Merge branch 'haypo-py3'

4 files changed, 96 insertions, 326 deletions

diff --git a/memcache.py b/memcache.py
index f5b9f49..5d97714 100644
--- a/memcache.py
+++ b/memcache.py
@@ -258,12 +258,12 @@ class Client(threading.local):
         return key
 
     def _encode_cmd(self, cmd, key, headers, noreply, *args):
-        cmd_bytes = cmd.encode() if six.PY3 else cmd
+        cmd_bytes = cmd.encode('utf-8') if six.PY3 else cmd
         fullcmd = [cmd_bytes, b' ', key]
 
         if headers:
             if six.PY3:
-                headers = headers.encode()
+                headers = headers.encode('utf-8')
             fullcmd.append(b' ')
             fullcmd.append(headers)
 
@@ -410,7 +410,10 @@ class Client(threading.local):
             if server.connect():
                 # print("(using server %s)" % server,)
                 return server, key
-            serverhash = serverHashFunction(str(serverhash) + str(i))
+            serverhash = str(serverhash) + str(i)
+            if isinstance(serverhash, six.text_type):
+                serverhash = serverhash.encode('ascii')
+            serverhash = serverHashFunction(serverhash)
         return None, None
 
     def disconnect_all(self):
@@ -456,15 +459,15 @@ class Client(threading.local):
         for server in six.iterkeys(server_keys):
             bigcmd = []
             write = bigcmd.append
-            extra = ' noreply' if noreply else ''
             if time is not None:
-                for key in server_keys[server]:  # These are mangled keys
-                    write("delete %s %d%s\r\n" % (key, time, extra))
+                headers = str(time)
             else:
-                for key in server_keys[server]:  # These are mangled keys
-                    write("delete %s%s\r\n" % (key, extra))
+                headers = None
+            for key in server_keys[server]:  # These are mangled keys
+                cmd = self._encode_cmd('delete', key, headers, noreply, b'\r\n')
+                write(cmd)
             try:
-                server.send_cmds(''.join(bigcmd))
+                server.send_cmds(b''.join(bigcmd))
             except socket.error as msg:
                 rc = 0
                 if isinstance(msg, tuple):
@@ -483,7 +486,7 @@ class Client(threading.local):
         for server, keys in six.iteritems(server_keys):
             try:
                 for key in keys:
-                    server.expect("DELETED")
+                    server.expect(b"DELETED")
             except socket.error as msg:
                 if isinstance(msg, tuple):
                     msg = msg[1]
@@ -528,9 +531,10 @@ class Client(threading.local):
             return 0
         self._statlog(cmd)
         if time is not None and time != 0:
-            fullcmd = self._encode_cmd(cmd, key, str(time), noreply)
+            headers = str(time)
         else:
-            fullcmd = self._encode_cmd(cmd, key, None, noreply)
+            headers = None
+        fullcmd = self._encode_cmd(cmd, key, headers, noreply)
 
         try:
             server.send_cmd(fullcmd)
@@ -905,12 +909,12 @@ class Client(threading.local):
 
         #  short-circuit if there are no servers, just return all keys
         if not server_keys:
-            return(mapping.keys())
+            return list(mapping.keys())
 
         for server, keys in six.iteritems(server_keys):
             try:
                 for key in keys:
-                    if server.readline() == 'STORED':
+                    if server.readline() == b'STORED':
                         continue
                     else:
                         # un-mangle.
@@ -1039,7 +1043,7 @@ class Client(threading.local):
             self._statlog(cmd)
 
             try:
-                cmd_bytes = cmd.encode() if six.PY3 else cmd
+                cmd_bytes = cmd.encode('utf-8') if six.PY3 else cmd
                 fullcmd = b''.join((cmd_bytes, b' ', key))
                 server.send_cmd(fullcmd)
                 rkey = flags = rlen = cas_id = None
@@ -1434,9 +1438,11 @@ class _Host(object):
         if self.debug and line != text:
             if six.PY3:
                 text = text.decode('utf8')
-                line = line.decode('utf8', 'replace')
+                log_line = line.decode('utf8', 'replace')
+            else:
+                log_line = line
             self.debuglog("while expecting %r, got unexpected response %r"
-                          % (text, line))
+                          % (text, log_line))
         return line
 
     def recv(self, rlen):
@@ -1472,236 +1478,13 @@ def _doctest():
     import doctest
     import memcache
     servers = ["127.0.0.1:11211"]
-    mc = Client(servers, debug=1)
+    mc = memcache.Client(servers, debug=1)
     globs = {"mc": mc}
-    return doctest.testmod(memcache, globs=globs)
-
-if __name__ == "__main__":
-    failures = 0
-    print("Testing docstrings...")
-    _doctest()
-    print("Running tests:")
-    print()
-    serverList = [["127.0.0.1:11211"]]
-    if '--do-unix' in sys.argv:
-        serverList.append([os.path.join(os.getcwd(), 'memcached.socket')])
-
-    for servers in serverList:
-        mc = Client(servers, debug=1)
-
-        def to_s(val):
-            if not isinstance(val, _str_cls):
-                return "%s (%s)" % (val, type(val))
-            return "%s" % val
-
-        def test_setget(key, val, noreply=False):
-            global failures
-            print("Testing set/get (noreply=%s) {'%s': %s} ..."
-                  % (noreply, to_s(key), to_s(val)), end=" ")
-            mc.set(key, val, noreply=noreply)
-            newval = mc.get(key)
-            if newval == val:
-                print("OK")
-                return 1
-            else:
-                print("FAIL")
-                failures += 1
-                return 0
-
-        class FooStruct(object):
-
-            def __init__(self):
-                self.bar = "baz"
-
-            def __str__(self):
-                return "A FooStruct"
-
-            def __eq__(self, other):
-                if isinstance(other, FooStruct):
-                    return self.bar == other.bar
-                return 0
-
-        test_setget("a_string", "some random string")
-        test_setget("a_string_2", "some random string", noreply=True)
-        test_setget("an_integer", 42)
-        test_setget("an_integer_2", 42, noreply=True)
-        if six.PY3:
-            ok = test_setget("long", 1 << 30)
-        else:
-            ok = test_setget("long", long(1 << 30))
-        if ok:
-            print("Testing delete ...", end=" ")
-            if mc.delete("long"):
-                print("OK")
-            else:
-                print("FAIL")
-                failures += 1
-            print("Checking results of delete ...", end=" ")
-            if mc.get("long") is None:
-                print("OK")
-            else:
-                print("FAIL")
-                failures += 1
-        print("Testing get_multi ...",)
-        print(mc.get_multi(["a_string", "an_integer", "a_string_2",
-                            "an_integer_2"]))
-
-        #  removed from the protocol
-        # if test_setget("timed_delete", 'foo'):
-        #     print "Testing timed delete ...",
-        #     if mc.delete("timed_delete", 1):
-        #         print("OK")
-        #     else:
-        #         print("FAIL")
-        #         failures += 1
-        #     print "Checking results of timed delete ..."
-        #     if mc.get("timed_delete") is None:
-        #         print("OK")
-        #     else:
-        #         print("FAIL")
-        #         failures += 1
-
-        print("Testing get(unknown value) ...", end=" ")
-        print(to_s(mc.get("unknown_value")))
-
-        f = FooStruct()
-        test_setget("foostruct", f)
-        test_setget("foostruct_2", f, noreply=True)
-
-        print("Testing incr ...", end=" ")
-        x = mc.incr("an_integer", 1)
-        if x == 43:
-            print("OK")
-        else:
-            print("FAIL")
-            failures += 1
-
-        print("Testing incr (noreply=True) ...", end=" ")
-        mc.incr("an_integer_2", 1, noreply=True)
-        x = mc.get("an_integer_2")
-        if x == 43:
-            print("OK")
-        else:
-            print("FAIL")
-            failures += 1
-
-        print("Testing decr ...", end=" ")
-        x = mc.decr("an_integer", 1)
-        if x == 42:
-            print("OK")
-        else:
-            print("FAIL")
-            failures += 1
-        sys.stdout.flush()
-
-        print("Testing decr (noreply=True) ...", end=" ")
-        mc.decr("an_integer_2", 1, noreply=True)
-        x = mc.get("an_integer_2")
-        if x == 42:
-            print("OK")
-        else:
-            print("FAIL")
-            failures += 1
-        sys.stdout.flush()
-
-        # sanity tests
-        print("Testing sending spaces...", end=" ")
-        sys.stdout.flush()
-        try:
-            x = mc.set("this has spaces", 1)
-        except Client.MemcachedKeyCharacterError as msg:
-            print("OK")
-        else:
-            print("FAIL")
-            failures += 1
-
-        print("Testing sending control characters...", end=" ")
-        try:
-            x = mc.set("this\x10has\x11control characters\x02", 1)
-        except Client.MemcachedKeyCharacterError as msg:
-            print("OK")
-        else:
-            print("FAIL")
-            failures += 1
-
-        print("Testing using insanely long key...", end=" ")
-        try:
-            x = mc.set('a'*SERVER_MAX_KEY_LENGTH, 1)
-            x = mc.set('a'*SERVER_MAX_KEY_LENGTH, 1, noreply=True)
-        except Client.MemcachedKeyLengthError as msg:
-            print("FAIL")
-            failures += 1
-        else:
-            print("OK")
-        try:
-            x = mc.set('a'*SERVER_MAX_KEY_LENGTH + 'a', 1)
-        except Client.MemcachedKeyLengthError as msg:
-            print("OK")
-        else:
-            print("FAIL")
-            failures += 1
-
-        print("Testing sending a unicode-string key...", end=" ")
-        try:
-            x = mc.set(unicode('keyhere'), 1)
-        except Client.MemcachedStringEncodingError as msg:
-            print("OK", end=" ")
-        else:
-            print("FAIL", end=" ")
-            failures += 1
-        try:
-            x = mc.set((unicode('a')*SERVER_MAX_KEY_LENGTH).encode('utf-8'), 1)
-        except Client.MemcachedKeyError:
-            print("FAIL", end=" ")
-            failures += 1
-        else:
-            print("OK", end=" ")
-        s = pickle.loads('V\\u4f1a\np0\n.')
-        try:
-            x = mc.set((s * SERVER_MAX_KEY_LENGTH).encode('utf-8'), 1)
-        except Client.MemcachedKeyLengthError:
-            print("OK")
-        else:
-            print("FAIL")
-            failures += 1
-
-        print("Testing using a value larger than the memcached value limit...")
-        print('NOTE: "MemCached: while expecting[...]" is normal...')
-        x = mc.set('keyhere', 'a'*SERVER_MAX_VALUE_LENGTH)
-        if mc.get('keyhere') is None:
-            print("OK", end=" ")
-        else:
-            print("FAIL", end=" ")
-            failures += 1
-        x = mc.set('keyhere', 'a'*SERVER_MAX_VALUE_LENGTH + 'aaa')
-        if mc.get('keyhere') is None:
-            print("OK")
-        else:
-            print("FAIL")
-            failures += 1
-
-        print("Testing set_multi() with no memcacheds running", end=" ")
-        mc.disconnect_all()
-        errors = mc.set_multi({'keyhere': 'a', 'keythere': 'b'})
-        if errors != []:
-            print("FAIL")
-            failures += 1
-        else:
-            print("OK")
-
-        print("Testing delete_multi() with no memcacheds running", end=" ")
-        mc.disconnect_all()
-        ret = mc.delete_multi({'keyhere': 'a', 'keythere': 'b'})
-        if ret != 1:
-            print("FAIL")
-            failures += 1
-        else:
-            print("OK")
-
-    if failures > 0:
-        print('*** THERE WERE FAILED TESTS')
+    results = doctest.testmod(memcache, globs=globs)
+    mc.disconnect_all()
+    print("Doctests: %s" % (results,))
+    if results.failed:
         sys.exit(1)
-    sys.exit(0)
 
 
 # vim: ts=4 sw=4 et :
diff --git a/tests/test_memcache.py b/tests/test_memcache.py
index ee60f1f..5e5d55f 100644
--- a/tests/test_memcache.py
+++ b/tests/test_memcache.py
@@ -1,10 +1,11 @@
 from __future__ import print_function
 
-from unittest import TestCase
+import time
+import unittest
 
 import six
 
-from memcache import Client, SERVER_MAX_KEY_LENGTH
+from memcache import Client, SERVER_MAX_KEY_LENGTH, SERVER_MAX_VALUE_LENGTH
 
 try:
     _str_cls = basestring
@@ -32,12 +33,14 @@ class FooStruct(object):
         return 0
 
 
-class TestMemcache(TestCase):
+class TestMemcache(unittest.TestCase):
     def setUp(self):
         # TODO: unix socket server stuff
         servers = ["127.0.0.1:11211"]
         self.mc = Client(servers, debug=1)
-        pass
+
+    def tearDown(self):
+        self.mc.disconnect_all()
 
     def check_setget(self, key, val, noreply=False):
         self.mc.set(key, val, noreply=noreply)
@@ -64,6 +67,8 @@ class TestMemcache(TestCase):
             {"gm_an_integer": 42, "gm_a_string": "some random string"})
 
     def test_get_unknown_value(self):
+        self.mc.delete("unknown_value")
+
         self.assertEqual(self.mc.get("unknown_value"), None)
 
     def test_setget_foostruct(self):
@@ -124,78 +129,56 @@ class TestMemcache(TestCase):
         """GitHub issue #75. Set/get with boolean values."""
         self.check_setget("bool", True)
 
-if __name__ == "__main__":
-    # failures = 0
-    # print("Testing docstrings...")
-    # _doctest()
-    # print("Running tests:")
-    # print()
-    # serverList = [["127.0.0.1:11211"]]
-    # if '--do-unix' in sys.argv:
-    #     serverList.append([os.path.join(os.getcwd(), 'memcached.socket')])
-
-    # for servers in serverList:
-    #     mc = Client(servers, debug=1)
-    if False:
-
-        print("Testing sending a unicode-string key...", end=" ")
-        try:
-            x = mc.set(six.u('keyhere'), 1)
-        except Client.MemcachedStringEncodingError as msg:
-            print("OK", end=" ")
-        else:
-            print("FAIL", end=" ")
-            failures += 1
-        try:
-            x = mc.set((six.u('a')*SERVER_MAX_KEY_LENGTH).encode('utf-8'), 1)
-        except Client.MemcachedKeyError:
-            print("FAIL", end=" ")
-            failures += 1
-        else:
-            print("OK", end=" ")
-        s = pickle.loads('V\\u4f1a\np0\n.')
-        try:
-            x = mc.set((s * SERVER_MAX_KEY_LENGTH).encode('utf-8'), 1)
-        except Client.MemcachedKeyLengthError:
-            print("OK")
-        else:
-            print("FAIL")
-            failures += 1
-
-        print("Testing using a value larger than the memcached value limit...")
-        print('NOTE: "MemCached: while expecting[...]" is normal...')
-        x = mc.set('keyhere', 'a'*SERVER_MAX_VALUE_LENGTH)
-        if mc.get('keyhere') is None:
-            print("OK", end=" ")
-        else:
-            print("FAIL", end=" ")
-            failures += 1
-        x = mc.set('keyhere', 'a'*SERVER_MAX_VALUE_LENGTH + 'aaa')
-        if mc.get('keyhere') is None:
-            print("OK")
-        else:
-            print("FAIL")
-            failures += 1
-
-        print("Testing set_multi() with no memcacheds running", end=" ")
-        mc.disconnect_all()
-        errors = mc.set_multi({'keyhere': 'a', 'keythere': 'b'})
-        if errors != []:
-            print("FAIL")
-            failures += 1
-        else:
-            print("OK")
-
-        print("Testing delete_multi() with no memcacheds running", end=" ")
-        mc.disconnect_all()
-        ret = mc.delete_multi({'keyhere': 'a', 'keythere': 'b'})
-        if ret != 1:
-            print("FAIL")
-            failures += 1
-        else:
-            print("OK")
 
-    if failures > 0:
-        print('*** THERE WERE FAILED TESTS')
-        sys.exit(1)
-    sys.exit(0)
+    def test_unicode_key(self):
+        s = six.u('\u4f1a')
+        maxlen = SERVER_MAX_KEY_LENGTH // len(s.encode('utf-8'))
+        key = s * maxlen
+
+        self.mc.set(key, 5)
+        value = self.mc.get(key)
+        self.assertEqual(value, 5)
+
+    def test_ignore_too_large_value(self):
+        # NOTE: "MemCached: while expecting[...]" is normal...
+        key = 'keyhere'
+
+        value = 'a' * (SERVER_MAX_VALUE_LENGTH // 2)
+        self.assertTrue(self.mc.set(key, value))
+        self.assertEqual(self.mc.get(key), value)
+
+        value = 'a' * SERVER_MAX_VALUE_LENGTH
+        self.assertFalse(self.mc.set(key, value))
+        # This test fails if the -I option is used on the memcached server
+        self.assertIsNone(self.mc.get(key))
+
+    def test_get_set_multi_key_prefix(self):
+        """Testing set_multi() with no memcacheds running."""
+
+        prefix = 'pfx_'
+        values = {'key1': 'a', 'key2': 'b'}
+        errors = self.mc.set_multi(values, key_prefix=prefix)
+        self.assertEqual(errors, [])
+
+        keys = list(values)
+        self.assertEqual(self.mc.get_multi(keys, key_prefix=prefix),
+                         values)
+
+    def test_set_multi_dead_servers(self):
+        """Testing set_multi() with no memcacheds running."""
+
+        self.mc.disconnect_all()
+        for server in self.mc.servers:
+            server.mark_dead('test')
+        errors = self.mc.set_multi({'key1': 'a', 'key2': 'b'})
+        self.assertEqual(sorted(errors), ['key1', 'key2'])
+
+    def test_disconnect_all_delete_multi(self):
+        """Testing delete_multi() with no memcacheds running."""
+        self.mc.disconnect_all()
+        ret = self.mc.delete_multi({'keyhere': 'a', 'keythere': 'b'})
+        self.assertEqual(ret, 1)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/test_setmulti.py b/tests/test_setmulti.py
index 2f7108a..39d43c9 100644
--- a/tests/test_setmulti.py
+++ b/tests/test_setmulti.py
@@ -53,18 +53,20 @@ class test_Memcached_Set_Multi(unittest.TestCase):
         self.old_socket = socket.socket
         socket.socket = FakeSocket
 
+        self.mc = memcache.Client(['memcached'], debug=True)
+
     def tearDown(self):
         socket.socket = self.old_socket
 
     def test_Socket_Disconnect(self):
-        client = memcache.Client(['memcached'], debug=True)
         mapping = {'foo': 'FOO', 'bar': 'BAR'}
-        bad_keys = client.set_multi(mapping)
+        bad_keys = self.mc.set_multi(mapping)
 
         self.assertEqual(sorted(bad_keys), ['bar', 'foo'])
 
         if DEBUG:
             print('set_multi({0!r}) -> {1!r}'.format(mapping, bad_keys))
 
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/tox.ini b/tox.ini
index 60348cb..444c527 100644
--- a/tox.ini
+++ b/tox.ini
@@ -9,7 +9,9 @@ usedevelop = True
 install_command = pip install -U {opts} {packages}
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
-commands = nosetests {posargs}
+commands =
+    nosetests {posargs}
+    python -c 'import memcache; memcache._doctest()'
 
 [tox:jenkins]
 downloadcache = ~/cache/pip