THRIFT-1857 Python 3 Support

Client: Python
Patch: Thomas Bartelmess, Eevee (Alex Munroe), helgridly, Christian Verkerk, Jeroen Vlek, Nobuaki Sukegawa

This closes #213 and closes #680

8 files changed, 58 insertions, 48 deletions

diff --git a/test/py/RunClientServer.py b/test/py/RunClientServer.py
index b7f43320a..7224bac91 100755
--- a/test/py/RunClientServer.py
+++ b/test/py/RunClientServer.py
@@ -20,6 +20,7 @@
 #
 
 from __future__ import division
+from __future__ import print_function
 import time
 import socket
 import subprocess
@@ -77,13 +78,13 @@ SERVERS = [
 try:
   import multiprocessing
 except:
-  print 'Warning: the multiprocessing module is unavailable. Skipping tests for TProcessPoolServer'
+  print('Warning: the multiprocessing module is unavailable. Skipping tests for TProcessPoolServer')
   SERVERS.remove('TProcessPoolServer')
 
 try:
   import ssl
 except:
-  print 'Warning, no ssl module available. Skipping all SSL tests.'
+  print('Warning, no ssl module available. Skipping all SSL tests.')
   SKIP_SSL.extend(SERVERS)
 
 # commandline permits a single class name to be specified to override SERVERS=[...]
@@ -91,7 +92,7 @@ if len(args) == 1:
   if args[0] in SERVERS:
     SERVERS = args
   else:
-    print 'Unavailable server type "%s", please choose one of: %s' % (args[0], SERVERS)
+    print('Unavailable server type "%s", please choose one of: %s' % (args[0], SERVERS))
     sys.exit(0)
 
 
@@ -102,7 +103,7 @@ def runScriptTest(genpydir, script):
   script_args = [sys.executable, relfile(script) ]
   script_args.append('--genpydir=%s' % genpydir)
   serverproc = subprocess.Popen(script_args)
-  print '\nTesting script: %s\n----' % (' '.join(script_args))
+  print('\nTesting script: %s\n----' % (' '.join(script_args)))
   ret = subprocess.call(script_args)
   if ret != 0:
     raise Exception("Script subprocess failed, retcode=%d, args: %s" % (ret, ' '.join(script_args)))
@@ -133,12 +134,12 @@ def runServiceTest(genpydir, server_class, proto, port, use_zlib, use_ssl):
   if server_class == 'THttpServer':
     cli_args.append('--http=/')
   if options.verbose > 0:
-    print 'Testing server %s: %s' % (server_class, ' '.join(server_args))
+    print('Testing server %s: %s' % (server_class, ' '.join(server_args)))
   serverproc = subprocess.Popen(server_args)
 
   def ensureServerAlive():
     if serverproc.poll() is not None:
-      print ('FAIL: Server process (%s) failed with retcode %d'
+      print(('FAIL: Server process (%s) failed with retcode %d')
              % (' '.join(server_args), serverproc.returncode))
       raise Exception('Server subprocess %s died, args: %s'
                       % (server_class, ' '.join(server_args)))
@@ -161,7 +162,7 @@ def runServiceTest(genpydir, server_class, proto, port, use_zlib, use_ssl):
 
   try:
     if options.verbose > 0:
-      print 'Testing client: %s' % (' '.join(cli_args))
+      print('Testing client: %s' % (' '.join(cli_args)))
     ret = subprocess.call(cli_args)
     if ret != 0:
       raise Exception("Client subprocess failed, retcode=%d, args: %s" % (ret, ' '.join(cli_args)))
@@ -170,7 +171,7 @@ def runServiceTest(genpydir, server_class, proto, port, use_zlib, use_ssl):
     ensureServerAlive()
     extra_sleep = EXTRA_DELAY.get(server_class, 0)
     if extra_sleep > 0 and options.verbose > 0:
-      print ('Giving %s (proto=%s,zlib=%s,ssl=%s) an extra %d seconds for child'
+      print('Giving %s (proto=%s,zlib=%s,ssl=%s) an extra %d seconds for child'
              'processes to terminate via alarm'
              % (server_class, proto, use_zlib, use_ssl, extra_sleep))
       time.sleep(extra_sleep)
@@ -179,22 +180,22 @@ def runServiceTest(genpydir, server_class, proto, port, use_zlib, use_ssl):
 
 test_count = 0
 # run tests without a client/server first
-print '----------------'
-print ' Executing individual test scripts with various generated code directories'
-print ' Directories to be tested: ' + ', '.join(generated_dirs)
-print ' Scripts to be tested: ' + ', '.join(SCRIPTS)
-print '----------------'
+print('----------------')
+print(' Executing individual test scripts with various generated code directories')
+print(' Directories to be tested: ' + ', '.join(generated_dirs))
+print(' Scripts to be tested: ' + ', '.join(SCRIPTS))
+print('----------------')
 for genpydir in generated_dirs:
   for script in SCRIPTS:
     runScriptTest(genpydir, script)
 
-print '----------------'
-print ' Executing Client/Server tests with various generated code directories'
-print ' Servers to be tested: ' + ', '.join(SERVERS)
-print ' Directories to be tested: ' + ', '.join(generated_dirs)
-print ' Protocols to be tested: ' + ', '.join(PROTOS)
-print ' Options to be tested: ZLIB(yes/no), SSL(yes/no)'
-print '----------------'
+print('----------------')
+print(' Executing Client/Server tests with various generated code directories')
+print(' Servers to be tested: ' + ', '.join(SERVERS))
+print(' Directories to be tested: ' + ', '.join(generated_dirs))
+print(' Protocols to be tested: ' + ', '.join(PROTOS))
+print(' Options to be tested: ZLIB(yes/no), SSL(yes/no)')
+print('----------------')
 for try_server in SERVERS:
   for genpydir in generated_dirs:
     for try_proto in PROTOS:
@@ -208,7 +209,7 @@ for try_server in SERVERS:
             continue
           test_count += 1
           if options.verbose > 0:
-            print '\nTest run #%d:  (includes %s) Server=%s,  Proto=%s,  zlib=%s,  SSL=%s' % (test_count, genpydir, try_server, try_proto, with_zlib, with_ssl)
+            print('\nTest run #%d:  (includes %s) Server=%s,  Proto=%s,  zlib=%s,  SSL=%s' % (test_count, genpydir, try_server, try_proto, with_zlib, with_ssl))
           runServiceTest(genpydir, try_server, try_proto, options.port, with_zlib, with_ssl)
           if options.verbose > 0:
-            print 'OK: Finished (includes %s)  %s / %s proto / zlib=%s / SSL=%s.   %d combinations tested.' % (genpydir, try_server, try_proto, with_zlib, with_ssl, test_count)
+            print('OK: Finished (includes %s)  %s / %s proto / zlib=%s / SSL=%s.   %d combinations tested.' % (genpydir, try_server, try_proto, with_zlib, with_ssl, test_count))
diff --git a/test/py/SerializationTest.py b/test/py/SerializationTest.py
index 40a52e67a..29f36564b 100755
--- a/test/py/SerializationTest.py
+++ b/test/py/SerializationTest.py
@@ -26,7 +26,7 @@ parser.add_option('--genpydir', type='string', dest='genpydir', default='gen-py'
 options, args = parser.parse_args()
 del sys.argv[1:] # clean up hack so unittest doesn't complain
 sys.path.insert(0, options.genpydir)
-sys.path.insert(0, glob.glob('../../lib/py/build/lib.*')[0])
+sys.path.insert(0, glob.glob('../../lib/py/build/lib*')[0])
 
 from ThriftTest.ttypes import *
 from DebugProtoTest.ttypes import CompactProtoTestStruct, Empty
@@ -305,7 +305,7 @@ class AcceleratedFramedTest(unittest.TestCase):
     prot.writeString(bigstring)
     prot.writeI16(24)
     data = databuf.getvalue()
-    cutpoint = len(data)/2
+    cutpoint = len(data) // 2
     parts = [ data[:cutpoint], data[cutpoint:] ]
 
     framed_buffer = TTransport.TMemoryBuffer()
@@ -346,14 +346,14 @@ class SerializersTest(unittest.TestCase):
     objcopy = Bools()
     deserialize(objcopy, serialize(obj))
     self.assertEquals(obj, objcopy)
-    
+
     # test enums
-    for num, name in Numberz._VALUES_TO_NAMES.iteritems():
+    for num, name in Numberz._VALUES_TO_NAMES.items():
       obj = Bonk(message='enum Numberz value %d is string %s' % (num, name), type=num)
       objcopy = Bonk()
       deserialize(objcopy, serialize(obj))
       self.assertEquals(obj, objcopy)
-  
+
 
 def suite():
   suite = unittest.TestSuite()
diff --git a/test/py/TSimpleJSONProtocolTest.py b/test/py/TSimpleJSONProtocolTest.py
index 080293a08..b8db93296 100644
--- a/test/py/TSimpleJSONProtocolTest.py
+++ b/test/py/TSimpleJSONProtocolTest.py
@@ -27,7 +27,7 @@ parser.add_option('--genpydir', type='string', dest='genpydir', default='gen-py'
 options, args = parser.parse_args()
 del sys.argv[1:] # clean up hack so unittest doesn't complain
 sys.path.insert(0, options.genpydir)
-sys.path.insert(0, glob.glob('../../lib/py/build/lib.*')[0])
+sys.path.insert(0, glob.glob('../../lib/py/build/lib*')[0])
 
 from ThriftTest.ttypes import *
 from thrift.protocol import TJSONProtocol
@@ -45,7 +45,7 @@ class SimpleJSONProtocolTest(unittest.TestCase):
       # assertDictEqual only in Python 2.7. Depends on your machine.
       self.assertDictEqual(a, b, msg)
       return
-    
+
     # Substitute implementation not as good as unittest library's
     self.assertEquals(len(a), len(b), msg)
     for k, v in a.iteritems():
@@ -66,7 +66,7 @@ class SimpleJSONProtocolTest(unittest.TestCase):
 
   def testWriteOnly(self):
     self.assertRaises(NotImplementedError,
-                      self._deserialize, VersioningTestV1, '{}')
+                      self._deserialize, VersioningTestV1, b'{}')
 
   def testSimpleMessage(self):
       v1obj = VersioningTestV1(
@@ -76,10 +76,10 @@ class SimpleJSONProtocolTest(unittest.TestCase):
       expected = dict(begin_in_both=v1obj.begin_in_both,
                       old_string=v1obj.old_string,
                       end_in_both=v1obj.end_in_both)
-      actual = json.loads(self._serialize(v1obj))
+      actual = json.loads(self._serialize(v1obj).decode('ascii'))
 
       self._assertDictEqual(expected, actual)
-     
+
   def testComplicated(self):
       v2obj = VersioningTestV2(
           begin_in_both=12345,
@@ -107,10 +107,10 @@ class SimpleJSONProtocolTest(unittest.TestCase):
                       newmap=v2obj.newmap,
                       newstring=v2obj.newstring,
                       end_in_both=v2obj.end_in_both)
-      
+
       # Need to load/dump because map keys get escaped.
       expected = json.loads(json.dumps(expected))
-      actual = json.loads(self._serialize(v2obj))
+      actual = json.loads(self._serialize(v2obj).decode('ascii'))
       self._assertDictEqual(expected, actual)
 
 
diff --git a/test/py/TestClient.py b/test/py/TestClient.py
index 3a74353f7..4689d634d 100755
--- a/test/py/TestClient.py
+++ b/test/py/TestClient.py
@@ -54,11 +54,11 @@ parser.set_defaults(framed=False, http_path=None, verbose=1, host='localhost', p
 options, args = parser.parse_args()
 
 script_dir = os.path.abspath(os.path.dirname(__file__))
-lib_dir = os.path.join(os.path.dirname(os.path.dirname(script_dir)), 'lib', 'py', 'build', 'lib.*')
+lib_dir = os.path.join(os.path.dirname(os.path.dirname(script_dir)), 'lib', 'py', 'build', 'lib*')
 sys.path.insert(0, os.path.join(script_dir, options.genpydir))
 sys.path.insert(0, glob.glob(lib_dir)[0])
 
-from ThriftTest import ThriftTest, SecondService
+from ThriftTest import ThriftTest
 from ThriftTest.ttypes import *
 from thrift.transport import TTransport
 from thrift.transport import TSocket
@@ -105,8 +105,8 @@ class AbstractTest(unittest.TestCase):
     print('testString')
     self.assertEqual(self.client.testString('Python' * 20), 'Python' * 20)
     self.assertEqual(self.client.testString(''), '')
-    self.assertEqual(self.client.testString(u'パイソン'.encode('utf8')), u'パイソン'.encode('utf8'))
-    s = u"""Afrikaans, Alemannisch, Aragonés, العربية, مصرى,
+    s1 = u'\b\t\n/\\\\\r{}:パイソン"'
+    s2 = u"""Afrikaans, Alemannisch, Aragonés, العربية, مصرى,
         Asturianu, Aymar aru, Azərbaycan, Башҡорт, Boarisch, Žemaitėška,
         Беларуская, Беларуская (тарашкевіца), Български, Bamanankan,
         বাংলা, Brezhoneg, Bosanski, Català, Mìng-dĕ̤ng-ngṳ̄, Нохчийн,
@@ -131,7 +131,11 @@ class AbstractTest(unittest.TestCase):
         Türkçe, Татарча/Tatarça, Українська, اردو, Tiếng Việt, Volapük,
         Walon, Winaray, 吴语, isiXhosa, ייִדיש, Yorùbá, Zeêuws, 中文,
         Bân-lâm-gú, 粵語"""
-    self.assertEqual(self.client.testString(s.encode('utf8')), s.encode('utf8'))
+    if sys.version_info[0] == 2:
+      s1 = s1.encode('utf8')
+      s2 = s2.encode('utf8')
+    self.assertEqual(self.client.testString(s1), s1)
+    self.assertEqual(self.client.testString(s2), s2)
 
   def testBool(self):
     print('testBool')
@@ -290,19 +294,23 @@ class AbstractTest(unittest.TestCase):
 class NormalBinaryTest(AbstractTest):
   protocol_factory = TBinaryProtocol.TBinaryProtocolFactory()
 
+
 class CompactTest(AbstractTest):
   protocol_factory = TCompactProtocol.TCompactProtocolFactory()
 
+
 class JSONTest(AbstractTest):
   protocol_factory = TJSONProtocol.TJSONProtocolFactory()
 
+
 class AcceleratedBinaryTest(AbstractTest):
   protocol_factory = TBinaryProtocol.TBinaryProtocolAcceleratedFactory()
 
+
 def suite():
   suite = unittest.TestSuite()
   loader = unittest.TestLoader()
-  if options.proto == 'binary': # look for --proto on cmdline
+  if options.proto == 'binary':  # look for --proto on cmdline
     suite.addTest(loader.loadTestsFromTestCase(NormalBinaryTest))
   elif options.proto == 'accel':
     suite.addTest(loader.loadTestsFromTestCase(AcceleratedBinaryTest))
@@ -314,6 +322,7 @@ def suite():
     raise AssertionError('Unknown protocol given with --protocol: %s' % options.proto)
   return suite
 
+
 class OwnArgsTestProgram(unittest.TestProgram):
     def parseArgs(self, argv):
         if args:
diff --git a/test/py/TestEof.py b/test/py/TestEof.py
index a9d81f1a9..7677de817 100755
--- a/test/py/TestEof.py
+++ b/test/py/TestEof.py
@@ -26,7 +26,7 @@ parser.add_option('--genpydir', type='string', dest='genpydir', default='gen-py'
 options, args = parser.parse_args()
 del sys.argv[1:] # clean up hack so unittest doesn't complain
 sys.path.insert(0, options.genpydir)
-sys.path.insert(0, glob.glob('../../lib/py/build/lib.*')[0])
+sys.path.insert(0, glob.glob('../../lib/py/build/lib*')[0])
 
 from ThriftTest import ThriftTest
 from ThriftTest.ttypes import *
@@ -99,7 +99,7 @@ class TestEof(unittest.TestCase):
     # TODO: we should make sure this covers more of the code paths
 
     data = self.make_data(pfactory)
-    for i in xrange(0, len(data) + 1):
+    for i in range(0, len(data) + 1):
       trans = TTransport.TMemoryBuffer(data[0:i])
       prot = pfactory.getProtocol(trans)
       try:
diff --git a/test/py/TestServer.py b/test/py/TestServer.py
index e7478e421..bc221c059 100755
--- a/test/py/TestServer.py
+++ b/test/py/TestServer.py
@@ -18,7 +18,7 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-from __future__ import division
+from __future__ import division, print_function
 import glob
 import logging
 import os
@@ -53,7 +53,7 @@ parser.set_defaults(port=9090, verbose=1, proto='binary')
 options, args = parser.parse_args()
 
 script_dir = os.path.realpath(os.path.dirname(__file__))  # <-- absolute dir the script is in
-lib_dir = os.path.join(os.path.dirname(os.path.dirname(script_dir)), 'lib', 'py', 'build', 'lib.*')
+lib_dir = os.path.join(os.path.dirname(os.path.dirname(script_dir)), 'lib', 'py', 'build', 'lib*')
 
 sys.path.insert(0, os.path.join(script_dir, options.genpydir))
 sys.path.insert(0, glob.glob(lib_dir)[0])
diff --git a/test/py/TestSocket.py b/test/py/TestSocket.py
index b9bdf27ea..55e4996e8 100755
--- a/test/py/TestSocket.py
+++ b/test/py/TestSocket.py
@@ -26,7 +26,7 @@ parser.add_option('--genpydir', type='string', dest='genpydir', default='gen-py'
 options, args = parser.parse_args()
 del sys.argv[1:] # clean up hack so unittest doesn't complain
 sys.path.insert(0, options.genpydir)
-sys.path.insert(0, glob.glob('../../lib/py/build/lib.*')[0])
+sys.path.insert(0, glob.glob('../../lib/py/build/lib*')[0])
 
 from ThriftTest import ThriftTest
 from ThriftTest.ttypes import *
@@ -41,7 +41,7 @@ from optparse import OptionParser
 
 class TimeoutTest(unittest.TestCase):
     def setUp(self):
-        for i in xrange(50):
+        for i in range(50):
             try:
                 # find a port we can use
                 self.listen_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
@@ -58,7 +58,7 @@ class TimeoutTest(unittest.TestCase):
 
         try:
             leaky = []
-            for i in xrange(100):
+            for i in range(100):
                 socket = TSocket.TSocket('localhost', self.port)
                 socket.setTimeout(10)
                 socket.open()
diff --git a/test/py/TestSyntax.py b/test/py/TestSyntax.py
index 9f71cf58c..cdf0e0da0 100755
--- a/test/py/TestSyntax.py
+++ b/test/py/TestSyntax.py
@@ -26,7 +26,7 @@ parser.add_option('--genpydir', type='string', dest='genpydir', default='gen-py'
 options, args = parser.parse_args()
 del sys.argv[1:] # clean up hack so unittest doesn't complain
 sys.path.insert(0, options.genpydir)
-sys.path.insert(0, glob.glob('../../lib/py/build/lib.*')[0])
+sys.path.insert(0, glob.glob('../../lib/py/build/lib*')[0])
 
 # Just import these generated files to make sure they are syntactically valid
 from DebugProtoTest import EmptyService