Merge branch 'drop-2to3'

15 files changed, 171 insertions, 84 deletions

diff --git a/NEWS b/NEWS
index 77d547c..3dd6cd9 100644
--- a/NEWS
+++ b/NEWS
@@ -7,15 +7,18 @@ What's new in psycopg 2.8
 New features:
 
 - Added `~psycopg2.extensions.encrypt_password()` function (:ticket:`#576`).
+- `~psycopg2.extras.DictCursor` and `~psycopg2.extras.RealDictCursor` rows
+  maintain columns order (:ticket:`#177`).
 
 Other changes:
 
 - Dropped support for Python 2.6, 3.2, 3.3.
 - Dropped `psycopg1` module.
 - Dropped deprecated ``register_tstz_w_secs()`` (was previously a no-op).
+- No longer use 2to3 during installation for Python 2 & 3 compatability. All
+  source files are now compatible with Python 2 & 3 as is.
 - The ``psycopg2.test`` package is no longer installed by ``python setup.py
-  install``. The test source files now are compatible with Python 2 and 3
-  without using 2to3.
+  install``.
 
 
 What's new in psycopg 2.7.5
diff --git a/lib/_ipaddress.py b/lib/_ipaddress.py
index ee05a26..beb2fb3 100644
--- a/lib/_ipaddress.py
+++ b/lib/_ipaddress.py
@@ -25,6 +25,7 @@
 
 from psycopg2.extensions import (
     new_type, new_array_type, register_type, register_adapter, QuotedString)
+from psycopg2.compat import text_type
 
 # The module is imported on register_ipaddress
 ipaddress = None
@@ -76,13 +77,13 @@ def cast_interface(s, cur=None):
     if s is None:
         return None
     # Py2 version force the use of unicode. meh.
-    return ipaddress.ip_interface(unicode(s))
+    return ipaddress.ip_interface(text_type(s))
 
 
 def cast_network(s, cur=None):
     if s is None:
         return None
-    return ipaddress.ip_network(unicode(s))
+    return ipaddress.ip_network(text_type(s))
 
 
 def adapt_ipaddress(obj):
diff --git a/lib/_range.py b/lib/_range.py
index c1facc0..fd15a76 100644
--- a/lib/_range.py
+++ b/lib/_range.py
@@ -29,6 +29,7 @@ import re
 from psycopg2._psycopg import ProgrammingError, InterfaceError
 from psycopg2.extensions import ISQLQuote, adapt, register_adapter
 from psycopg2.extensions import new_type, new_array_type, register_type
+from psycopg2.compat import string_types
 
 
 class Range(object):
@@ -126,9 +127,13 @@ class Range(object):
 
         return True
 
-    def __nonzero__(self):
+    def __bool__(self):
         return self._bounds is not None
 
+    def __nonzero__(self):
+        # Python 2 compatibility
+        return type(self).__bool__(self)
+
     def __eq__(self, other):
         if not isinstance(other, Range):
             return False
@@ -296,7 +301,7 @@ class RangeCaster(object):
         # an implementation detail and is not documented. It is currently used
         # for the numeric ranges.
         self.adapter = None
-        if isinstance(pgrange, basestring):
+        if isinstance(pgrange, string_types):
             self.adapter = type(pgrange, (RangeAdapter,), {})
             self.adapter.name = pgrange
         else:
@@ -313,7 +318,7 @@ class RangeCaster(object):
 
         self.range = None
         try:
-            if isinstance(pyrange, basestring):
+            if isinstance(pyrange, string_types):
                 self.range = type(pyrange, (Range,), {})
             if issubclass(pyrange, Range) and pyrange is not Range:
                 self.range = pyrange
diff --git a/lib/compat.py b/lib/compat.py
new file mode 100644
index 0000000..cfd5a88
--- /dev/null
+++ b/lib/compat.py
@@ -0,0 +1,10 @@
+import sys
+
+if sys.version_info[0] == 2:
+    # Python 2
+    string_types = basestring,
+    text_type = unicode
+else:
+    # Python 3
+    string_types = str,
+    text_type = str
diff --git a/lib/errorcodes.py b/lib/errorcodes.py
index 24fcf25..b8742f5 100644
--- a/lib/errorcodes.py
+++ b/lib/errorcodes.py
@@ -40,7 +40,7 @@ def lookup(code, _cache={}):
 
     # Generate the lookup map at first usage.
     tmp = {}
-    for k, v in globals().iteritems():
+    for k, v in globals().items():
         if isinstance(v, str) and len(v) in (2, 5):
             tmp[v] = k
 
diff --git a/lib/extensions.py b/lib/extensions.py
index 3c0e225..3661e6c 100644
--- a/lib/extensions.py
+++ b/lib/extensions.py
@@ -163,7 +163,7 @@ def make_dsn(dsn=None, **kwargs):
         kwargs['dbname'] = kwargs.pop('database')
 
     # Drop the None arguments
-    kwargs = {k: v for (k, v) in kwargs.iteritems() if v is not None}
+    kwargs = {k: v for (k, v) in kwargs.items() if v is not None}
 
     if dsn is not None:
         tmp = parse_dsn(dsn)
@@ -171,7 +171,7 @@ def make_dsn(dsn=None, **kwargs):
         kwargs = tmp
 
     dsn = " ".join(["%s=%s" % (k, _param_escape(str(v)))
-        for (k, v) in kwargs.iteritems()])
+        for (k, v) in kwargs.items()])
 
     # verify that the returned dsn is valid
     parse_dsn(dsn)
@@ -216,7 +216,7 @@ del Range
 # When the encoding is set its name is cleaned up from - and _ and turned
 # uppercase, so an encoding not respecting these rules wouldn't be found in the
 # encodings keys and would raise an exception with the unicode typecaster
-for k, v in encodings.items():
+for k, v in list(encodings.items()):
     k = k.replace('_', '').replace('-', '').upper()
     encodings[k] = v
 
diff --git a/lib/extras.py b/lib/extras.py
index 9c26ccb..ff32ab6 100644
--- a/lib/extras.py
+++ b/lib/extras.py
@@ -29,7 +29,7 @@ import os as _os
 import sys as _sys
 import time as _time
 import re as _re
-from collections import namedtuple
+from collections import namedtuple, OrderedDict
 
 try:
     import logging as _logging
@@ -140,12 +140,12 @@ class DictCursor(DictCursorBase):
         self._prefetch = 1
 
     def execute(self, query, vars=None):
-        self.index = {}
+        self.index = OrderedDict()
         self._query_executed = 1
         return super(DictCursor, self).execute(query, vars)
 
     def callproc(self, procname, vars=None):
-        self.index = {}
+        self.index = OrderedDict()
         self._query_executed = 1
         return super(DictCursor, self).callproc(procname, vars)
 
@@ -168,24 +168,23 @@ class DictRow(list):
     def __getitem__(self, x):
         if not isinstance(x, (int, slice)):
             x = self._index[x]
-        return list.__getitem__(self, x)
+        return super(DictRow, self).__getitem__(x)
 
     def __setitem__(self, x, v):
         if not isinstance(x, (int, slice)):
             x = self._index[x]
-        list.__setitem__(self, x, v)
+        super(DictRow, self).__setitem__(x, v)
 
     def items(self):
-        return list(self.iteritems())
+        g = super(DictRow, self).__getitem__
+        return ((n, g(self._index[n])) for n in self._index)
 
     def keys(self):
-        return self._index.keys()
+        return iter(self._index)
 
     def values(self):
-        return tuple(self[:])
-
-    def has_key(self, x):
-        return x in self._index
+        g = super(DictRow, self).__getitem__
+        return (g(self._index[n]) for n in self._index)
 
     def get(self, x, default=None):
         try:
@@ -193,18 +192,8 @@ class DictRow(list):
         except:
             return default
 
-    def iteritems(self):
-        for n, v in self._index.iteritems():
-            yield n, list.__getitem__(self, v)
-
-    def iterkeys(self):
-        return self._index.iterkeys()
-
-    def itervalues(self):
-        return list.__iter__(self)
-
     def copy(self):
-        return dict(self.iteritems())
+        return OrderedDict(self.items())
 
     def __contains__(self, x):
         return x in self._index
@@ -216,12 +205,20 @@ class DictRow(list):
         self[:] = data[0]
         self._index = data[1]
 
-    # drop the crusty Py2 methods
-    if _sys.version_info[0] > 2:
-        items = iteritems               # noqa
-        keys = iterkeys                 # noqa
-        values = itervalues             # noqa
-        del iteritems, iterkeys, itervalues, has_key
+    if _sys.version_info[0] < 3:
+        iterkeys = keys
+        itervalues = values
+        iteritems = items
+        has_key = __contains__
+
+        def keys(self):
+            return list(self.iterkeys())
+
+        def values(self):
+            return tuple(self.itervalues())
+
+        def items(self):
+            return list(self.iteritems())
 
 
 class RealDictConnection(_connection):
@@ -256,8 +253,7 @@ class RealDictCursor(DictCursorBase):
 
     def _build_index(self):
         if self._query_executed == 1 and self.description:
-            for i in range(len(self.description)):
-                self.column_mapping.append(self.description[i][0])
+            self.column_mapping = [d[0] for d in self.description]
             self._query_executed = 0
 
 
@@ -267,7 +263,7 @@ class RealDictRow(dict):
     __slots__ = ('_column_mapping',)
 
     def __init__(self, cursor):
-        dict.__init__(self)
+        super(RealDictRow, self).__init__()
         # Required for named cursors
         if cursor.description and not cursor.column_mapping:
             cursor._build_index()
@@ -277,7 +273,7 @@ class RealDictRow(dict):
     def __setitem__(self, name, value):
         if type(name) == int:
             name = self._column_mapping[name]
-        return dict.__setitem__(self, name, value)
+        super(RealDictRow, self).__setitem__(name, value)
 
     def __getstate__(self):
         return self.copy(), self._column_mapping[:]
@@ -286,6 +282,32 @@ class RealDictRow(dict):
         self.update(data[0])
         self._column_mapping = data[1]
 
+    def __iter__(self):
+        return iter(self._column_mapping)
+
+    def keys(self):
+        return iter(self._column_mapping)
+
+    def values(self):
+        return (self[k] for k in self._column_mapping)
+
+    def items(self):
+        return ((k, self[k]) for k in self._column_mapping)
+
+    if _sys.version_info[0] < 3:
+        iterkeys = keys
+        itervalues = values
+        iteritems = items
+
+        def keys(self):
+            return list(self.iterkeys())
+
+        def values(self):
+            return list(self.itervalues())
+
+        def items(self):
+            return list(self.iteritems())
+
 
 class NamedTupleConnection(_connection):
     """A connection that uses `NamedTupleCursor` automatically."""
@@ -337,14 +359,14 @@ class NamedTupleCursor(_cursor):
         nt = self.Record
         if nt is None:
             nt = self.Record = self._make_nt()
-        return map(nt._make, ts)
+        return list(map(nt._make, ts))
 
     def fetchall(self):
         ts = super(NamedTupleCursor, self).fetchall()
         nt = self.Record
         if nt is None:
             nt = self.Record = self._make_nt()
-        return map(nt._make, ts)
+        return list(map(nt._make, ts))
 
     def __iter__(self):
         try:
@@ -598,7 +620,7 @@ class ReplicationCursor(_replicationCursor):
                     "cannot specify output plugin options for physical replication")
 
             command += " ("
-            for k, v in options.iteritems():
+            for k, v in options.items():
                 if not command.endswith('('):
                     command += ", "
                 command += "%s %s" % (quote_ident(k, self), _A(str(v)))
@@ -794,7 +816,7 @@ class HstoreAdapter(object):
 
         adapt = _ext.adapt
         rv = []
-        for k, v in self.wrapped.iteritems():
+        for k, v in self.wrapped.items():
             k = adapt(k)
             k.prepare(self.conn)
             k = k.getquoted()
@@ -816,9 +838,9 @@ class HstoreAdapter(object):
         if not self.wrapped:
             return b"''::hstore"
 
-        k = _ext.adapt(self.wrapped.keys())
+        k = _ext.adapt(list(self.wrapped.keys()))
         k.prepare(self.conn)
-        v = _ext.adapt(self.wrapped.values())
+        v = _ext.adapt(list(self.wrapped.values()))
         v.prepare(self.conn)
         return b"hstore(" + k.getquoted() + b", " + v.getquoted() + b")"
 
@@ -1144,7 +1166,7 @@ def _paginate(seq, page_size):
     it = iter(seq)
     while 1:
         try:
-            for i in xrange(page_size):
+            for i in range(page_size):
                 page.append(next(it))
             yield page
             page = []
diff --git a/lib/pool.py b/lib/pool.py
index a91c9cc..6c26f7d 100644
--- a/lib/pool.py
+++ b/lib/pool.py
@@ -209,8 +209,8 @@ class PersistentConnectionPool(AbstractConnectionPool):
 
         # we we'll need the thread module, to determine thread ids, so we
         # import it here and copy it in an instance variable
-        import thread as _thread  # work around for 2to3 bug - see ticket #348
-        self.__thread = _thread
+        import thread
+        self.__thread = thread
 
     def getconn(self):
         """Generate thread id and return a connection."""
diff --git a/lib/sql.py b/lib/sql.py
index 849b25f..7ba9295 100644
--- a/lib/sql.py
+++ b/lib/sql.py
@@ -27,6 +27,7 @@ import sys
 import string
 
 from psycopg2 import extensions as ext
+from psycopg2.compat import string_types
 
 
 _formatter = string.Formatter()
@@ -147,7 +148,7 @@ class Composed(Composable):
             "foo", "bar"
 
         """
-        if isinstance(joiner, basestring):
+        if isinstance(joiner, string_types):
             joiner = SQL(joiner)
         elif not isinstance(joiner, SQL):
             raise TypeError(
@@ -179,7 +180,7 @@ class SQL(Composable):
         select "foo", "bar" from "table"
     """
     def __init__(self, string):
-        if not isinstance(string, basestring):
+        if not isinstance(string, string_types):
             raise TypeError("SQL values must be strings")
         super(SQL, self).__init__(string)
 
@@ -308,7 +309,7 @@ class Identifier(Composable):
 
     """
     def __init__(self, string):
-        if not isinstance(string, basestring):
+        if not isinstance(string, string_types):
             raise TypeError("SQL identifiers must be strings")
 
         super(Identifier, self).__init__(string)
@@ -395,7 +396,7 @@ class Placeholder(Composable):
     """
 
     def __init__(self, name=None):
-        if isinstance(name, basestring):
+        if isinstance(name, string_types):
             if ')' in name:
                 raise ValueError("invalid name: %r" % name)
 
diff --git a/scripts/make_errorcodes.py b/scripts/make_errorcodes.py
index adc6ee8..aa5e4cd 100755
--- a/scripts/make_errorcodes.py
+++ b/scripts/make_errorcodes.py
@@ -159,7 +159,7 @@ def fetch_errors(versions):
         # https://github.com/postgres/postgres/commit/28e0727076
         errors['55']['55P04'] = 'UNSAFE_NEW_ENUM_VALUE_USAGE'
 
-        for c, cerrs in e1.iteritems():
+        for c, cerrs in e1.items():
             errors[c].update(cerrs)
 
     return classes, errors
diff --git a/setup.py b/setup.py
index 2d608c8..f2e8260 100644
--- a/setup.py
+++ b/setup.py
@@ -43,21 +43,6 @@ from distutils.errors import CompileError
 from distutils.util import get_platform
 
 try:
-    from distutils.command.build_py import build_py_2to3
-except ImportError:
-    from distutils.command.build_py import build_py
-else:
-    class build_py(build_py_2to3):
-        # workaround subclass for ticket #153
-        pass
-
-    # Configure distutils to run our custom 2to3 fixers as well
-    from lib2to3.refactor import get_fixers_from_package
-    build_py.fixer_names = [f for f in get_fixers_from_package('lib2to3.fixes')
-        # creates a pending deprecation warning on py 3.4
-        if not f.endswith('.fix_reload')]
-
-try:
     import configparser
 except ImportError:
     import ConfigParser as configparser
@@ -661,7 +646,5 @@ setup(name="psycopg2",
       data_files=data_files,
       package_dir={'psycopg2': 'lib'},
       packages=['psycopg2'],
-      cmdclass={
-          'build_ext': psycopg_build_ext,
-          'build_py': build_py, },
+      cmdclass={'build_ext': psycopg_build_ext},
       ext_modules=ext)
diff --git a/tests/test_cursor.py b/tests/test_cursor.py
index b3e03d9..b48fe7f 100755
--- a/tests/test_cursor.py
+++ b/tests/test_cursor.py
@@ -29,9 +29,10 @@ import psycopg2.extensions
 import unittest
 from .testutils import (ConnectingTestCase, skip_before_postgres,
     skip_if_no_getrefcount, slow, skip_if_no_superuser,
-    skip_if_windows, unicode)
+    skip_if_windows)
 
 import psycopg2.extras
+from psycopg2.compat import text_type
 
 
 class CursorTests(ConnectingTestCase):
@@ -75,7 +76,7 @@ class CursorTests(ConnectingTestCase):
         snowman = u"\u2603"
 
         def b(s):
-            if isinstance(s, unicode):
+            if isinstance(s, text_type):
                 return s.encode('utf8')
             else:
                 return s
diff --git a/tests/test_extras_dictcursor.py b/tests/test_extras_dictcursor.py
index a9201f1..1d5dfd0 100755
--- a/tests/test_extras_dictcursor.py
+++ b/tests/test_extras_dictcursor.py
@@ -15,6 +15,7 @@
 # License for more details.
 
 import time
+import pickle
 from datetime import timedelta
 import psycopg2
 import psycopg2.extras
@@ -140,7 +141,6 @@ class ExtrasDictCursorTests(_DictCursorBase):
         self.failUnless(row[0] == 'bar')
 
     def testPickleDictRow(self):
-        import pickle
         curs = self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
         curs.execute("select 10 as a, 20 as b")
         r = curs.fetchone()
@@ -184,6 +184,37 @@ class ExtrasDictCursorTests(_DictCursorBase):
         self.assert_(not isinstance(r.items(), list))
         self.assertEqual(len(list(r.items())), 2)
 
+    def test_order(self):
+        curs = self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
+        curs.execute("select 5 as foo, 4 as bar, 33 as baz, 2 as qux")
+        r = curs.fetchone()
+        self.assertEqual(list(r), [5, 4, 33, 2])
+        self.assertEqual(list(r.keys()), ['foo', 'bar', 'baz', 'qux'])
+        self.assertEqual(list(r.values()), [5, 4, 33, 2])
+        self.assertEqual(list(r.items()),
+            [('foo', 5), ('bar', 4), ('baz', 33), ('qux', 2)])
+
+        r1 = pickle.loads(pickle.dumps(r))
+        self.assertEqual(list(r1), list(r))
+        self.assertEqual(list(r1.keys()), list(r.keys()))
+        self.assertEqual(list(r1.values()), list(r.values()))
+        self.assertEqual(list(r1.items()), list(r.items()))
+
+    @skip_from_python(3)
+    def test_order_iter(self):
+        curs = self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
+        curs.execute("select 5 as foo, 4 as bar, 33 as baz, 2 as qux")
+        r = curs.fetchone()
+        self.assertEqual(list(r.iterkeys()), ['foo', 'bar', 'baz', 'qux'])
+        self.assertEqual(list(r.itervalues()), [5, 4, 33, 2])
+        self.assertEqual(list(r.iteritems()),
+            [('foo', 5), ('bar', 4), ('baz', 33), ('qux', 2)])
+
+        r1 = pickle.loads(pickle.dumps(r))
+        self.assertEqual(list(r1.iterkeys()), list(r.iterkeys()))
+        self.assertEqual(list(r1.itervalues()), list(r.itervalues()))
+        self.assertEqual(list(r1.iteritems()), list(r.iteritems()))
+
 
 class ExtrasDictCursorRealTests(_DictCursorBase):
     def testDictCursorWithPlainCursorRealFetchOne(self):
@@ -216,7 +247,6 @@ class ExtrasDictCursorRealTests(_DictCursorBase):
         self.failUnless(row['foo'] == 'bar')
 
     def testPickleRealDictRow(self):
-        import pickle
         curs = self.conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
         curs.execute("select 10 as a, 20 as b")
         r = curs.fetchone()
@@ -293,6 +323,37 @@ class ExtrasDictCursorRealTests(_DictCursorBase):
         self.assert_(not isinstance(r.items(), list))
         self.assertEqual(len(list(r.items())), 2)
 
+    def test_order(self):
+        curs = self.conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
+        curs.execute("select 5 as foo, 4 as bar, 33 as baz, 2 as qux")
+        r = curs.fetchone()
+        self.assertEqual(list(r), ['foo', 'bar', 'baz', 'qux'])
+        self.assertEqual(list(r.keys()), ['foo', 'bar', 'baz', 'qux'])
+        self.assertEqual(list(r.values()), [5, 4, 33, 2])
+        self.assertEqual(list(r.items()),
+            [('foo', 5), ('bar', 4), ('baz', 33), ('qux', 2)])
+
+        r1 = pickle.loads(pickle.dumps(r))
+        self.assertEqual(list(r1), list(r))
+        self.assertEqual(list(r1.keys()), list(r.keys()))
+        self.assertEqual(list(r1.values()), list(r.values()))
+        self.assertEqual(list(r1.items()), list(r.items()))
+
+    @skip_from_python(3)
+    def test_order_iter(self):
+        curs = self.conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
+        curs.execute("select 5 as foo, 4 as bar, 33 as baz, 2 as qux")
+        r = curs.fetchone()
+        self.assertEqual(list(r.iterkeys()), ['foo', 'bar', 'baz', 'qux'])
+        self.assertEqual(list(r.itervalues()), [5, 4, 33, 2])
+        self.assertEqual(list(r.iteritems()),
+            [('foo', 5), ('bar', 4), ('baz', 33), ('qux', 2)])
+
+        r1 = pickle.loads(pickle.dumps(r))
+        self.assertEqual(list(r1.iterkeys()), list(r.iterkeys()))
+        self.assertEqual(list(r1.itervalues()), list(r.itervalues()))
+        self.assertEqual(list(r1.iteritems()), list(r.iteritems()))
+
 
 class NamedTupleCursorTest(ConnectingTestCase):
     def setUp(self):
diff --git a/tests/test_sql.py b/tests/test_sql.py
index 1c20997..81b22a4 100755
--- a/tests/test_sql.py
+++ b/tests/test_sql.py
@@ -26,10 +26,11 @@ import datetime as dt
 import unittest
 from .testutils import (ConnectingTestCase,
     skip_before_postgres, skip_before_python, skip_copy_if_green,
-    unicode, StringIO)
+    StringIO)
 
 import psycopg2
 from psycopg2 import sql
+from psycopg2.compat import text_type
 
 
 class SqlFormatTests(ConnectingTestCase):
@@ -64,7 +65,7 @@ class SqlFormatTests(ConnectingTestCase):
         s = sql.SQL(u"select {0} from {1}").format(
             sql.Identifier(u'field'), sql.Identifier('table'))
         s1 = s.as_string(self.conn)
-        self.assert_(isinstance(s1, unicode))
+        self.assert_(isinstance(s1, text_type))
         self.assertEqual(s1, u'select "field" from "table"')
 
     def test_compose_literal(self):
diff --git a/tests/testutils.py b/tests/testutils.py
index d70e091..3bb72e2 100644
--- a/tests/testutils.py
+++ b/tests/testutils.py
@@ -30,6 +30,7 @@ import platform
 import unittest
 from functools import wraps
 from .testconfig import dsn, repl_dsn
+from psycopg2.compat import text_type
 
 # Python 2/3 compatibility
 
@@ -39,14 +40,12 @@ if sys.version_info[0] == 2:
     long = long
     reload = reload
     unichr = unichr
-    unicode = unicode
 else:
     # Python 3
     from io import StringIO
     from importlib import reload
     long = int
     unichr = chr
-    unicode = str
 
 
 # Silence warnings caused by the stubbornness of the Python unittest
@@ -89,7 +88,7 @@ class ConnectingTestCase(unittest.TestCase):
     def assertQuotedEqual(self, first, second, msg=None):
         """Compare two quoted strings disregarding eventual E'' quotes"""
         def f(s):
-            if isinstance(s, unicode):
+            if isinstance(s, text_type):
                 return re.sub(r"\bE'", "'", s)
             elif isinstance(first, bytes):
                 return re.sub(br"\bE'", b"'", s)