Port routes to Python 3

The code now works on Python 2 and Python 3 without modifications: 2to3
is no more needed. Changes:

* Drop 2to3 from setup.py: the code is now directly compatible with
  Python 2 and Python 3
* Add dependency to six
* Drop support for Python 3.2: remove 3.2 from .travis.yml
* Replace "for key, value in dict.iteritems():" with "for key, value in
  six.iteritems(dict):"
* Use six.moves.urllib to get urllib functions
* Replace unicode() with six.text_type()
* Replace dict.keys() with list(dict.keys())
* Replace "dict.has_key(key)" with "key in dict"
* Remove "py3where=build" from notests section of setup.cfg
* Add parenthesis to print()
* Replace dict.items() with list(dict.items())

13 files changed, 91 insertions, 87 deletions

diff --git a/.travis.yml b/.travis.yml
index ca3b9c9..e698e42 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,7 +2,6 @@ language: python
 python:
   - "2.6"
   - "2.7"
-  - "3.2"
   - "3.3"
   - "3.4"
   - "pypy"
diff --git a/routes/mapper.py b/routes/mapper.py
index ea55cda..583309e 100644
--- a/routes/mapper.py
+++ b/routes/mapper.py
@@ -3,6 +3,7 @@ import re
 import threading
 
 from repoze.lru import LRUCache
+import six
 
 from routes import request_config
 from routes.util import (
@@ -153,7 +154,7 @@ class SubMapper(SubMapperParent):
     def connect(self, *args, **kwargs):
         newkargs = {}
         newargs = args
-        for key, value in self.kwargs.items():
+        for key, value in six.iteritems(self.kwargs):
             if key == 'path_prefix':
                 if len(args) > 1:
                     newargs = (args[0], self.kwargs[key] + args[1])
@@ -547,8 +548,8 @@ class Mapper(SubMapperParent):
         # Setup the lists of all controllers/actions we'll add each route
         # to. We include the '*' in the case that a generate contains a
         # controller/action that has no hardcodes
-        controllerlist = controllerlist.keys() + ['*']
-        actionlist = actionlist.keys() + ['*']
+        controllerlist = list(controllerlist.keys()) + ['*']
+        actionlist = list(actionlist.keys()) + ['*']
 
         # Go through our list again, assemble the controllers/actions we'll
         # add each route to. If its hardcoded, we only add it to that dict key.
@@ -562,7 +563,7 @@ class Mapper(SubMapperParent):
             if 'controller' in route.hardcoded:
                 clist = [route.defaults['controller']]
             if 'action' in route.hardcoded:
-                alist = [unicode(route.defaults['action'])]
+                alist = [six.text_type(route.defaults['action'])]
             for controller in clist:
                 for action in alist:
                     actiondict = gendict.setdefault(controller, {})
@@ -592,7 +593,7 @@ class Mapper(SubMapperParent):
             else:
                 clist = self.controller_scan
 
-        for key, val in self.maxkeys.iteritems():
+        for key, val in six.iteritems(self.maxkeys):
             for route in val:
                 route.makeregexp(clist)
 
@@ -758,8 +759,8 @@ class Mapper(SubMapperParent):
         # If the URL didn't depend on the SCRIPT_NAME, we'll cache it
         # keyed by just by kargs; otherwise we need to cache it with
         # both SCRIPT_NAME and kargs:
-        cache_key = unicode(args).encode('utf8') + \
-            unicode(kargs).encode('utf8')
+        cache_key = six.text_type(args).encode('utf8') + \
+            six.text_type(kargs).encode('utf8')
 
         if self.urlcache is not None:
             cache_key_script_name = '%s:%s' % (script_name, cache_key)
@@ -782,7 +783,7 @@ class Mapper(SubMapperParent):
 
         keys = frozenset(kargs.keys())
         cacheset = False
-        cachekey = unicode(keys)
+        cachekey = six.text_type(keys)
         cachelist = sortcache.get(cachekey)
         if args:
             keylist = args
@@ -1047,7 +1048,7 @@ class Mapper(SubMapperParent):
         def swap(dct, newdct):
             """Swap the keys and values in the dict, and uppercase the values
             from the dict during the swap."""
-            for key, val in dct.iteritems():
+            for key, val in six.iteritems(dct):
                 newdct.setdefault(val.upper(), []).append(key)
             return newdct
         collection_methods = swap(collection, {})
@@ -1088,7 +1089,7 @@ class Mapper(SubMapperParent):
             return opts
 
         # Add the routes for handling collection methods
-        for method, lst in collection_methods.iteritems():
+        for method, lst in six.iteritems(collection_methods):
             primary = (method != 'GET' and lst.pop(0)) or None
             route_options = requirements_for(method)
             for action in lst:
@@ -1112,7 +1113,7 @@ class Mapper(SubMapperParent):
                      action='index', conditions={'method': ['GET']}, **options)
 
         # Add the routes that deal with new resource methods
-        for method, lst in new_methods.iteritems():
+        for method, lst in six.iteritems(new_methods):
             route_options = requirements_for(method)
             for action in lst:
                 name = "new_" + member_name
@@ -1131,7 +1132,7 @@ class Mapper(SubMapperParent):
         requirements_regexp = '[^\/]+(?<!\\\)'
 
         # Add the routes that deal with member methods of a resource
-        for method, lst in member_methods.iteritems():
+        for method, lst in six.iteritems(member_methods):
             route_options = requirements_for(method)
             route_options['requirements'] = {'id': requirements_regexp}
             if method not in ['POST', 'GET', 'any']:
diff --git a/routes/route.py b/routes/route.py
index 901b4f6..ef8810c 100644
--- a/routes/route.py
+++ b/routes/route.py
@@ -1,10 +1,12 @@
 import re
 import sys
-import urllib
-
+from six.moves import urllib
 if sys.version < '2.4':
     from sets import ImmutableSet as frozenset
 
+import six
+from six.moves.urllib import parse as urlparse
+
 from routes.util import _url_quote as url_quote, _str_encode, as_unicode
 
 
@@ -87,18 +89,18 @@ class Route(object):
     def _setup_route(self):
         # Build our routelist, and the keys used in the route
         self.routelist = routelist = self._pathkeys(self.routepath)
-        routekeys = frozenset([key['name'] for key in routelist
-                               if isinstance(key, dict)])
-        self.dotkeys = frozenset([key['name'] for key in routelist
-                                  if isinstance(key, dict) and
-                                  key['type'] == '.'])
+        routekeys = frozenset(key['name'] for key in routelist
+                              if isinstance(key, dict))
+        self.dotkeys = frozenset(key['name'] for key in routelist
+                                 if isinstance(key, dict) and
+                                 key['type'] == '.')
 
         if not self.minimization:
             self.make_full_route()
 
         # Build a req list with all the regexp requirements for our args
         self.req_regs = {}
-        for key, val in self.reqs.iteritems():
+        for key, val in six.iteritems(self.reqs):
             self.req_regs[key] = re.compile('^' + val + '$')
         # Update our defaults and set new default keys if needed. defaults
         # needs to be saved
@@ -114,9 +116,9 @@ class Route(object):
 
         # Populate our hardcoded keys, these are ones that are set and don't
         # exist in the route
-        self.hardcoded = frozenset(
-            [key for key in self.maxkeys if key not in routekeys and
-             self.defaults[key] is not None])
+        self.hardcoded = frozenset(key for key in self.maxkeys
+                                   if key not in routekeys
+                                      and self.defaults[key] is not None)
 
         # Cache our default keys
         self._default_keys = frozenset(self.defaults.keys())
@@ -134,14 +136,14 @@ class Route(object):
 
     def make_unicode(self, s):
         """Transform the given argument into a unicode string."""
-        if isinstance(s, unicode):
+        if isinstance(s, six.text_type):
             return s
         elif isinstance(s, bytes):
             return s.decode(self.encoding)
         elif callable(s):
             return s
         else:
-            return unicode(s)
+            return six.text_type(s)
 
     def _pathkeys(self, routepath):
         """Utility function to walk the route, and pull out the valid
@@ -253,8 +255,8 @@ class Route(object):
         if 'action' not in routekeys and 'action' not in kargs \
            and not self.explicit:
             kargs['action'] = 'index'
-        defaultkeys = frozenset([key for key in kargs.keys()
-                                 if key not in reserved_keys])
+        defaultkeys = frozenset(key for key in kargs.keys()
+                                if key not in reserved_keys)
         for key in defaultkeys:
             if kargs[key] is not None:
                 defaults[key] = self.make_unicode(kargs[key])
@@ -266,8 +268,8 @@ class Route(object):
         if 'id' in routekeys and 'id' not in defaults \
            and not self.explicit:
             defaults['id'] = None
-        newdefaultkeys = frozenset([key for key in defaults.keys()
-                                    if key not in reserved_keys])
+        newdefaultkeys = frozenset(key for key in defaults.keys()
+                                   if key not in reserved_keys)
 
         return (defaults, newdefaultkeys)
 
@@ -554,7 +556,7 @@ class Route(object):
         matchdict = match.groupdict()
         result = {}
         extras = self._default_keys - frozenset(matchdict.keys())
-        for key, val in matchdict.iteritems():
+        for key, val in six.iteritems(matchdict):
             if key != 'path_info' and self.encoding:
                 # change back into python unicode objects from the URL
                 # representation
@@ -745,7 +747,7 @@ class Route(object):
                     fragments.append((key, _str_encode(val, self.encoding)))
             if fragments:
                 url += '?'
-                url += urllib.urlencode(fragments)
+                url += urlparse.urlencode(fragments)
         elif _append_slash and not url.endswith('/'):
             url += '/'
         return url
diff --git a/routes/util.py b/routes/util.py
index 9834ad5..baeeac7 100644
--- a/routes/util.py
+++ b/routes/util.py
@@ -7,7 +7,10 @@ framework.
 """
 import os
 import re
-import urllib
+
+import six
+from six.moves import urllib
+
 from routes import request_config
 
 
@@ -31,8 +34,8 @@ def _screenargs(kargs, mapper, environ, force_explicit=False):
     """
     # Coerce any unicode args with the encoding
     encoding = mapper.encoding
-    for key, val in kargs.iteritems():
-        if isinstance(val, unicode):
+    for key, val in six.iteritems(kargs):
+        if isinstance(val, six.text_type):
             kargs[key] = val.encode(encoding)
 
     if mapper.explicit and mapper.sub_domains and not force_explicit:
@@ -46,7 +49,7 @@ def _screenargs(kargs, mapper, environ, force_explicit=False):
         # If the controller name starts with '/', ignore route memory
         kargs['controller'] = kargs['controller'][1:]
         return kargs
-    elif controller_name and not kargs.has_key('action'):
+    elif controller_name and 'action' not in kargs:
         # Fill in an action if we don't have one, but have a controller
         kargs['action'] = 'index'
 
@@ -57,10 +60,10 @@ def _screenargs(kargs, mapper, environ, force_explicit=False):
         memory_kargs = {}
 
     # Remove keys from memory and kargs if kargs has them as None
-    for key in [key for key in kargs.keys() if kargs[key] is None]:
+    empty_keys = [key for key, value in six.iteritems(kargs) if value is None]
+    for key in empty_keys:
         del kargs[key]
-        if memory_kargs.has_key(key):
-            del memory_kargs[key]
+        memory_kargs.pop(key, None)
 
     # Merge the new args on top of the memory args
     memory_kargs.update(kargs)
@@ -76,7 +79,7 @@ def _subdomain_check(kargs, mapper, environ):
     on the current subdomain or lack therof."""
     if mapper.sub_domains:
         subdomain = kargs.pop('sub_domain', None)
-        if isinstance(subdomain, unicode):
+        if isinstance(subdomain, six.text_type):
             subdomain = str(subdomain)
 
         fullhost = environ.get('HTTP_HOST') or environ.get('SERVER_NAME')
@@ -107,27 +110,27 @@ def _subdomain_check(kargs, mapper, environ):
 def _url_quote(string, encoding):
     """A Unicode handling version of urllib.quote."""
     if encoding:
-        if isinstance(string, unicode):
+        if isinstance(string, six.text_type):
             s = string.encode(encoding)
-        elif isinstance(string, str):
+        elif isinstance(string, six.text_type):
             # assume the encoding is already correct
             s = string
         else:
-            s = unicode(string).encode(encoding)
+            s = six.text_type(string).encode(encoding)
     else:
         s = str(string)
-    return urllib.quote(s, '/')
+    return urllib.parse.quote(s, '/')
 
 
 def _str_encode(string, encoding):
     if encoding:
-        if isinstance(string, unicode):
+        if isinstance(string, six.text_type):
             s = string.encode(encoding)
-        elif isinstance(string, str):
+        elif isinstance(string, six.text_type):
             # assume the encoding is already correct
             s = string
         else:
-            s = unicode(string).encode(encoding)
+            s = six.text_type(string).encode(encoding)
     return s
 
 
@@ -207,16 +210,16 @@ def url_for(*args, **kargs):
             if kargs:
                 url += '?'
                 query_args = []
-                for key, val in kargs.iteritems():
+                for key, val in six.iteritems(kargs):
                     if isinstance(val, (list, tuple)):
                         for value in val:
                             query_args.append("%s=%s" % (
-                                urllib.quote(unicode(key).encode(encoding)),
-                                urllib.quote(unicode(value).encode(encoding))))
+                                urllib.parse.quote(six.text_type(key).encode(encoding)),
+                                urllib.parse.quote(six.text_type(value).encode(encoding))))
                     else:
                         query_args.append("%s=%s" % (
-                            urllib.quote(unicode(key).encode(encoding)),
-                            urllib.quote(unicode(val).encode(encoding))))
+                            urllib.parse.quote(six.text_type(key).encode(encoding)),
+                            urllib.parse.quote(six.text_type(val).encode(encoding))))
                 url += '&'.join(query_args)
     environ = getattr(config, 'environ', {})
     if 'wsgiorg.routing_args' not in environ:
@@ -352,16 +355,16 @@ class URLGenerator(object):
                 if kargs:
                     url += '?'
                     query_args = []
-                    for key, val in kargs.iteritems():
+                    for key, val in six.iteritems(kargs):
                         if isinstance(val, (list, tuple)):
                             for value in val:
                                 query_args.append("%s=%s" % (
-                                    urllib.quote(unicode(key).encode(encoding)),
-                                    urllib.quote(unicode(value).encode(encoding))))
+                                    urllib.parse.quote(six.text_type(key).encode(encoding)),
+                                    urllib.parse.quote(six.text_type(value).encode(encoding))))
                         else:
                             query_args.append("%s=%s" % (
-                                urllib.quote(unicode(key).encode(encoding)),
-                                urllib.quote(unicode(val).encode(encoding))))
+                                urllib.parse.quote(six.text_type(key).encode(encoding)),
+                                urllib.parse.quote(six.text_type(val).encode(encoding))))
                     url += '&'.join(query_args)
         if not static:
             route_args = []
diff --git a/setup.cfg b/setup.cfg
index 4134fbe..44b2824 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -15,4 +15,3 @@ cover-html=True
 cover-html-dir=html_coverage
 #cover-tests=True
 cover-package=routes
-py3where=build
diff --git a/setup.py b/setup.py
index c58bf94..5b9b268 100644
--- a/setup.py
+++ b/setup.py
@@ -18,7 +18,6 @@ extra_options = {
     }
 
 if PY3:
-    extra_options["use_2to3"] = True
     if "test" in sys.argv or "develop" in sys.argv:
         for root, directories, files in os.walk("tests"):
             for directory in directories:
@@ -54,6 +53,7 @@ setup(name="Routes",
       zip_safe=False,
       tests_require=['nose', 'webtest', 'webob', 'coverage'],
       install_requires=[
+          "six",
           "repoze.lru>=0.3"
       ],
       **extra_options
diff --git a/tests/test_functional/profile_rec.py b/tests/test_functional/profile_rec.py
index 2ac0ae7..1020a58 100644
--- a/tests/test_functional/profile_rec.py
+++ b/tests/test_functional/profile_rec.py
@@ -56,9 +56,9 @@ def bench_rec(mapper, n):
     end = time.time()
     total = end-start-(en-ts)
     per_url = total / (n*10)
-    print "Hit recognition\n"
-    print "%s ms/url" % (per_url*1000)
-    print "%s urls/s\n" % (1.00/per_url)
+    print("Hit recognition\n")
+    print("%s ms/url" % (per_url*1000))
+    print("%s urls/s\n" % (1.00/per_url))
 
     # misses
     start = time.time()
@@ -69,9 +69,9 @@ def bench_rec(mapper, n):
     end = time.time()
     total = end-start-(en-ts)
     per_url = total / (n*10)
-    print "Miss recognition\n"
-    print "%s ms/url" % (per_url*1000)
-    print "%s urls/s\n" % (1.00/per_url)
+    print("Miss recognition\n")
+    print("%s ms/url" % (per_url*1000))
+    print("%s urls/s\n" % (1.00/per_url))
         
 def do_profile(cmd, globals, locals, sort_order, callers):
     fd, fn = tempfile.mkstemp()
diff --git a/tests/test_functional/test_generation.py b/tests/test_functional/test_generation.py
index 5e6d51e..b461b5f 100644
--- a/tests/test_functional/test_generation.py
+++ b/tests/test_functional/test_generation.py
@@ -1,6 +1,6 @@
 """test_generation"""
 import sys, time, unittest
-import urllib
+from six.moves import urllib
 
 from nose.tools import eq_, assert_raises
 from routes import *
@@ -628,7 +628,7 @@ class TestGeneration(unittest.TestCase):
 
     def test_unicode(self):
         hoge = u'\u30c6\u30b9\u30c8' # the word test in Japanese
-        hoge_enc = urllib.quote(hoge.encode('utf-8'))
+        hoge_enc = urllib.parse.quote(hoge.encode('utf-8'))
         m = Mapper()
         m.connect(':hoge')
         eq_("/%s" % hoge_enc, m.generate(hoge=hoge))
@@ -636,7 +636,7 @@ class TestGeneration(unittest.TestCase):
 
     def test_unicode_static(self):
         hoge = u'\u30c6\u30b9\u30c8' # the word test in Japanese
-        hoge_enc = urllib.quote(hoge.encode('utf-8'))
+        hoge_enc = urllib.parse.quote(hoge.encode('utf-8'))
         m = Mapper()
         m.minimization = True
         m.connect('google-jp', 'http://www.google.co.jp/search', _static=True)
@@ -711,6 +711,6 @@ else:
         en = time.time()
         total = end-start-(en-ts)
         per_url = total / (n*6)
-        print "Generation (%s URLs)" % (n*6)
-        print "%s ms/url" % (per_url*1000)
-        print "%s urls/s\n" % (1.00/per_url)
+        print("Generation (%s URLs)" % (n*6))
+        print("%s ms/url" % (per_url*1000))
+        print("%s urls/s\n" % (1.00/per_url))
diff --git a/tests/test_functional/test_middleware.py b/tests/test_functional/test_middleware.py
index 000d383..882df9e 100644
--- a/tests/test_functional/test_middleware.py
+++ b/tests/test_functional/test_middleware.py
@@ -6,7 +6,7 @@ from nose.tools import eq_
 def simple_app(environ, start_response):
     route_dict = environ['wsgiorg.routing_args'][1]
     start_response('200 OK', [('Content-type', 'text/plain')])
-    items = route_dict.items()
+    items = list(route_dict.items())
     items.sort()
     return [('The matchdict items are %s and environ is %s' % (items, environ)).encode()]
 
@@ -87,14 +87,14 @@ def test_path_info():
     assert 'matchdict items are []' in res
 
     res = app.get('/myapp/some/other/url')
-    print res
+    print(res)
     assert b"matchdict items are [('action', " + repr(u'index').encode() + \
            b"), ('controller', " + repr(u'myapp').encode() + b"), ('path_info', 'some/other/url')]" in res
     assert "'SCRIPT_NAME': '/myapp'" in res
     assert "'PATH_INFO': '/some/other/url'" in res
 
     res = app.get('/project/pylonshq/browser/pylons/templates/default_project/+package+/pylonshq/browser/pylons/templates/default_project/+package+/controllers')
-    print res
+    print(res)
     assert "'SCRIPT_NAME': '/project'" in res
     assert "'PATH_INFO': '/pylonshq/browser/pylons/templates/default_project/+package+/pylonshq/browser/pylons/templates/default_project/+package+/controllers'" in res
 
@@ -115,7 +115,7 @@ def test_redirect_middleware():
     eq_(res.headers['Location'], '/static/faq/home.html')
 
     res = app.get('/myapp/some/other/url')
-    print res
+    print(res)
     assert b"matchdict items are [('action', " + repr(u'index').encode() + \
            b"), ('controller', " + repr(u'myapp').encode() + \
            b"), ('path_info', 'some/other/url')]" in res
diff --git a/tests/test_functional/test_nonminimization.py b/tests/test_functional/test_nonminimization.py
index e076f00..1b152c4 100644
--- a/tests/test_functional/test_nonminimization.py
+++ b/tests/test_functional/test_nonminimization.py
@@ -1,5 +1,5 @@
 """Test non-minimization recognition"""
-import urllib
+from six.moves import urllib
 
 from nose.tools import eq_
 
@@ -116,7 +116,7 @@ def test_regexp_syntax():
 
 def test_unicode():
     hoge = u'\u30c6\u30b9\u30c8' # the word test in Japanese
-    hoge_enc = urllib.quote(hoge.encode('utf-8'))
+    hoge_enc = urllib.parse.quote(hoge.encode('utf-8'))
     m = Mapper()
     m.minimization = False
     m.connect(':hoge')
@@ -125,7 +125,7 @@ def test_unicode():
 
 def test_unicode_static():
     hoge = u'\u30c6\u30b9\u30c8' # the word test in Japanese
-    hoge_enc = urllib.quote(hoge.encode('utf-8'))
+    hoge_enc = urllib.parse.quote(hoge.encode('utf-8'))
     m = Mapper()
     m.minimization = False
     m.connect('google-jp', 'http://www.google.co.jp/search', _static=True)
diff --git a/tests/test_functional/test_recognition.py b/tests/test_functional/test_recognition.py
index bec757c..5e50a60 100644
--- a/tests/test_functional/test_recognition.py
+++ b/tests/test_functional/test_recognition.py
@@ -3,7 +3,7 @@
 import sys
 import time
 import unittest
-import urllib
+from six.moves import urllib
 from nose.tools import eq_, assert_raises
 from routes import *
 from routes.util import RoutesException
@@ -45,7 +45,7 @@ class TestRecognition(unittest.TestCase):
 
     def test_disabling_unicode(self):
         hoge = u'\u30c6\u30b9\u30c8' # the word test in Japanese
-        hoge_enc = urllib.quote(hoge.encode('utf-8'))
+        hoge_enc = urllib.parse.quote(hoge.encode('utf-8'))
         m = Mapper(explicit=False)
         m.minimization = True
         m.encoding = None
@@ -989,9 +989,9 @@ else:
         en = time.time()
         total = end-start-(en-ts)
         per_url = total / (n*10)
-        print "Recognition\n"
-        print "%s ms/url" % (per_url*1000)
-        print "%s urls/s\n" % (1.00/per_url)
+        print("Recognition\n")
+        print("%s ms/url" % (per_url*1000))
+        print("%s urls/s\n" % (1.00/per_url))
 
 """
 Copyright (c) 2005 Ben Bangert <ben@groovie.org>, Parachute
diff --git a/tests/test_functional/test_utils.py b/tests/test_functional/test_utils.py
index 1f7d67e..3f4dd47 100644
--- a/tests/test_functional/test_utils.py
+++ b/tests/test_functional/test_utils.py
@@ -952,6 +952,6 @@ else:
         en = time.time()
         total = end-start-(en-ts)
         per_url = total / (n*6)
-        print "Generation (%s URLs) RouteSet" % (n*6)
-        print "%s ms/url" % (per_url*1000)
-        print "%s urls/s\n" % (1.00/per_url)
+        print("Generation (%s URLs) RouteSet" % (n*6))
+        print("%s ms/url" % (per_url*1000))
+        print("%s urls/s\n" % (1.00/per_url))
diff --git a/tests/test_units/test_environment.py b/tests/test_units/test_environment.py
index eceec03..c88a76b 100644
--- a/tests/test_units/test_environment.py
+++ b/tests/test_units/test_environment.py
@@ -28,7 +28,7 @@ class TestEnvironment(unittest.TestCase):
         assert con.mapper.environ == env
         assert con.protocol == 'http'
         assert con.host == 'somewhere.com'
-        assert con.mapper_dict.has_key('controller')
+        assert 'controller' in con.mapper_dict
         assert con.mapper_dict['controller'] == 'content'
 
 if __name__ == '__main__':