Switch from MySQL-python to PyMySQL

As discussed in the Liberty Design Summit "Moving apps to Python 3"
cross-project workshop, the way forward in the near future is to
switch to the pure-python PyMySQL library as a default.

Added a special test environment to keep MySQL-python support.
Documentation modified.

https://etherpad.openstack.org/p/liberty-cross-project-python3

Change-Id: I12b32dc097a121bd43991bc38dd4d289b65e86c1

11 files changed, 188 insertions, 100 deletions

diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
index fdacbf4..78528ec 100644
--- a/CONTRIBUTING.rst
+++ b/CONTRIBUTING.rst
@@ -26,12 +26,15 @@ How to run unit tests
 oslo.db (as all OpenStack projects) uses tox to run unit tests. You can find
 general information about OpenStack unit tests and testing with tox in wiki_.
 
-oslo.db tests use MySQL-python as the default MySQL DB API driver (which is
-true for OpenStack), and psycopg2 for PostgreSQL. pip will build these libs in
-your venv, so you must ensure that you have the required system packages
-installed.  For Ubuntu/Debian they are python-dev, libmysqlclient-dev and
-libpq-dev.  For Fedora/CentOS - gcc, python-devel, postgresql-devel and
-mysql-devel.
+oslo.db tests use PyMySQL as the default MySQL DB API driver (which is true for
+OpenStack), and psycopg2 for PostgreSQL. pip will build these libs in your
+venv, so you must ensure that you have the required system packages installed
+for psycopg2 (PyMySQL is a pure-Python implementation and so needs no
+additional system packages).  For Ubuntu/Debian they are python-dev, and
+libpq-dev.  For Fedora/CentOS - gcc, python-devel and postgresql-devel.
+There is also a separate env for testing with MySQL-python. If you are suppose
+to run these tests as well, you need to install libmysqlclient-dev on Ubuntu/Debian
+or mysql-devel for Fedora/CentOS.
 
 The oslo.db unit tests system allows to run unittests on real databases. At the
 moment it supports MySQL, PostgreSQL and SQLite.
diff --git a/doc/source/installation.rst b/doc/source/installation.rst
index 96e0ec2..1262160 100644
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -8,16 +8,37 @@ At the command line::
 
 You will also need to install at least one SQL backend::
 
-    $ pip install MySQL-python
+    $ pip install psycopg2
+
+Or::
+
+    $ pip install PyMySQL
 
 Or::
 
     $ pip install pysqlite
 
-Using with MySQL
-----------------
 
-If using MySQL make sure to install the MySQL client development package for
+Using with PostgreSQL
+---------------------
+
+If you are using PostgreSQL make sure to install the PostgreSQL client
+development package for your distro. On Ubuntu this is done as follows::
+
+    $ sudo apt-get install libpq-dev
+    $ pip install psycopg2
+
+The installation of psycopg2 will fail if libpq-dev is not installed first.
+Note that even in a virtual environment the libpq-dev will be installed
+system wide.
+
+
+Using with MySQL-python
+-----------------------
+
+PyMySQL is a default MySQL DB API driver for oslo.db, as well as for the whole
+OpenStack. But you still can use MySQL-python as an alternative DB API driver.
+For MySQL-python you must install the MySQL client development package for
 your distro. On Ubuntu this is done as follows::
 
     $ sudo apt-get install libmysqlclient-dev
diff --git a/oslo_db/sqlalchemy/provision.py b/oslo_db/sqlalchemy/provision.py
index 0062782..654678a 100644
--- a/oslo_db/sqlalchemy/provision.py
+++ b/oslo_db/sqlalchemy/provision.py
@@ -455,7 +455,7 @@ class MySQLBackendImpl(BackendImpl):
     default_engine_kwargs = {'mysql_sql_mode': 'TRADITIONAL'}
 
     def create_opportunistic_driver_url(self):
-        return "mysql://openstack_citest:openstack_citest@localhost/"
+        return "mysql+pymysql://openstack_citest:openstack_citest@localhost/"
 
     def create_named_database(self, engine, ident, conditional=False):
         with engine.connect() as conn:
diff --git a/oslo_db/tests/old_import_api/sqlalchemy/test_exc_filters.py b/oslo_db/tests/old_import_api/sqlalchemy/test_exc_filters.py
index 7738254..f8e09cc 100644
--- a/oslo_db/tests/old_import_api/sqlalchemy/test_exc_filters.py
+++ b/oslo_db/tests/old_import_api/sqlalchemy/test_exc_filters.py
@@ -46,7 +46,16 @@ class _SQLAExceptionMatcher(object):
             self.assertTrue(issubclass(exc.__class__, exception_type))
         else:
             self.assertEqual(exc.__class__.__name__, exception_type)
-        self.assertEqual(str(exc.orig).lower(), message.lower())
+        if isinstance(message, tuple):
+            self.assertEqual(
+                [a.lower()
+                 if isinstance(a, six.string_types) else a
+                 for a in exc.orig.args],
+                [m.lower()
+                 if isinstance(m, six.string_types) else m for m in message]
+            )
+        else:
+            self.assertEqual(str(exc.orig).lower(), message.lower())
         if sql is not None:
             self.assertEqual(exc.statement, sql)
         if params is not None:
@@ -359,10 +368,10 @@ class TestReferenceErrorMySQL(TestReferenceErrorSQLite,
         self.assertInnerException(
             matched,
             "IntegrityError",
-            "(1452, 'Cannot add or update a child row: a "
-            "foreign key constraint fails (`{0}`.`resource_entity`, "
-            "CONSTRAINT `foo_fkey` FOREIGN KEY (`foo_id`) REFERENCES "
-            "`resource_foo` (`id`))')".format(self.engine.url.database),
+            (1452, "Cannot add or update a child row: a "
+             "foreign key constraint fails (`{0}`.`resource_entity`, "
+             "CONSTRAINT `foo_fkey` FOREIGN KEY (`foo_id`) REFERENCES "
+             "`resource_foo` (`id`))".format(self.engine.url.database)),
             "INSERT INTO resource_entity (id, foo_id) VALUES (%s, %s)",
             (1, 2)
         )
@@ -382,10 +391,13 @@ class TestReferenceErrorMySQL(TestReferenceErrorSQLite,
         self.assertInnerException(
             matched,
             "IntegrityError",
-            '(1452, \'Cannot add or update a child row: a '
-            'foreign key constraint fails ("{0}"."resource_entity", '
-            'CONSTRAINT "foo_fkey" FOREIGN KEY ("foo_id") REFERENCES '
-            '"resource_foo" ("id"))\')'.format(self.engine.url.database),
+            (
+                1452,
+                'Cannot add or update a child row: a '
+                'foreign key constraint fails ("{0}"."resource_entity", '
+                'CONSTRAINT "foo_fkey" FOREIGN KEY ("foo_id") REFERENCES '
+                '"resource_foo" ("id"))'.format(self.engine.url.database)
+            ),
             "INSERT INTO resource_entity (id, foo_id) VALUES (%s, %s)",
             (1, 2)
         )
@@ -436,7 +448,7 @@ class TestDuplicate(TestsExceptionFilter):
             "PRIMARY KEY must be unique 'insert into t values(10)'",
             expected_columns=[])
 
-    def test_mysql_mysqldb(self):
+    def test_mysql_pymysql(self):
         self._run_dupe_constraint_test(
             "mysql",
             '(1062, "Duplicate entry '
@@ -541,7 +553,7 @@ class TestDeadlock(TestsExceptionFilter):
 
         self.assertEqual(matched.orig.__class__.__name__, expected_dbapi_cls)
 
-    def test_mysql_mysqldb_deadlock(self):
+    def test_mysql_pymysql_deadlock(self):
         self._run_deadlock_detect_test(
             "mysql",
             "(1213, 'Deadlock found when trying "
@@ -549,7 +561,7 @@ class TestDeadlock(TestsExceptionFilter):
             "transaction')"
         )
 
-    def test_mysql_mysqldb_galera_deadlock(self):
+    def test_mysql_pymysql_galera_deadlock(self):
         self._run_deadlock_detect_test(
             "mysql",
             "(1205, 'Lock wait timeout exceeded; "
diff --git a/oslo_db/tests/old_import_api/sqlalchemy/test_utils.py b/oslo_db/tests/old_import_api/sqlalchemy/test_utils.py
index 9c488a2..d748dac 100644
--- a/oslo_db/tests/old_import_api/sqlalchemy/test_utils.py
+++ b/oslo_db/tests/old_import_api/sqlalchemy/test_utils.py
@@ -793,8 +793,10 @@ class TestDialectFunctionDispatcher(test_base.BaseTestCase):
         dispatcher = orig = utils.dispatch_for_dialect("*")(
             callable_fn.default)
         dispatcher = dispatcher.dispatch_for("sqlite")(callable_fn.sqlite)
-        dispatcher = dispatcher.dispatch_for("mysql+mysqldb")(
-            callable_fn.mysql_mysqldb)
+        dispatcher = dispatcher.dispatch_for("mysql+pymysql")(
+            callable_fn.mysql_pymysql)
+        dispatcher = dispatcher.dispatch_for("mysql")(
+            callable_fn.mysql)
         dispatcher = dispatcher.dispatch_for("postgresql")(
             callable_fn.postgresql)
 
@@ -808,7 +810,8 @@ class TestDialectFunctionDispatcher(test_base.BaseTestCase):
         for targ in [
             callable_fn.default,
             callable_fn.sqlite,
-            callable_fn.mysql_mysqldb,
+            callable_fn.mysql,
+            callable_fn.mysql_pymysql,
             callable_fn.postgresql,
             callable_fn.postgresql_psycopg2,
             callable_fn.pyodbc
@@ -818,8 +821,10 @@ class TestDialectFunctionDispatcher(test_base.BaseTestCase):
         dispatcher = orig = utils.dispatch_for_dialect("*", multiple=True)(
             callable_fn.default)
         dispatcher = dispatcher.dispatch_for("sqlite")(callable_fn.sqlite)
-        dispatcher = dispatcher.dispatch_for("mysql+mysqldb")(
-            callable_fn.mysql_mysqldb)
+        dispatcher = dispatcher.dispatch_for("mysql+pymysql")(
+            callable_fn.mysql_pymysql)
+        dispatcher = dispatcher.dispatch_for("mysql")(
+            callable_fn.mysql)
         dispatcher = dispatcher.dispatch_for("postgresql+*")(
             callable_fn.postgresql)
         dispatcher = dispatcher.dispatch_for("postgresql+psycopg2")(
@@ -836,15 +841,17 @@ class TestDialectFunctionDispatcher(test_base.BaseTestCase):
         dispatcher, callable_fn = self._single_fixture()
         dispatcher("sqlite://", 1)
         dispatcher("postgresql+psycopg2://u:p@h/t", 2)
-        dispatcher("mysql://u:p@h/t", 3)
-        dispatcher("mysql+mysqlconnector://u:p@h/t", 4)
+        dispatcher("mysql+pymysql://u:p@h/t", 3)
+        dispatcher("mysql://u:p@h/t", 4)
+        dispatcher("mysql+mysqlconnector://u:p@h/t", 5)
 
         self.assertEqual(
             [
                 mock.call.sqlite('sqlite://', 1),
                 mock.call.postgresql("postgresql+psycopg2://u:p@h/t", 2),
-                mock.call.mysql_mysqldb("mysql://u:p@h/t", 3),
-                mock.call.default("mysql+mysqlconnector://u:p@h/t", 4)
+                mock.call.mysql_pymysql("mysql+pymysql://u:p@h/t", 3),
+                mock.call.mysql("mysql://u:p@h/t", 4),
+                mock.call.mysql("mysql+mysqlconnector://u:p@h/t", 5),
             ],
             callable_fn.mock_calls)
 
@@ -962,10 +969,10 @@ class TestDialectFunctionDispatcher(test_base.BaseTestCase):
 
     def test_single_retval(self):
         dispatcher, callable_fn = self._single_fixture()
-        callable_fn.mysql_mysqldb.return_value = 5
+        callable_fn.mysql_pymysql.return_value = 5
 
         self.assertEqual(
-            dispatcher("mysql://u:p@h/t", 3), 5
+            dispatcher("mysql+pymysql://u:p@h/t", 3), 5
         )
 
     def test_engine(self):
@@ -978,14 +985,25 @@ class TestDialectFunctionDispatcher(test_base.BaseTestCase):
             callable_fn.mock_calls
         )
 
-    def test_url(self):
+    def test_url_pymysql(self):
         url = sqlalchemy.engine.url.make_url(
-            "mysql+mysqldb://scott:tiger@localhost/test")
+            "mysql+pymysql://scott:tiger@localhost/test")
         dispatcher, callable_fn = self._single_fixture()
 
         dispatcher(url, 15)
         self.assertEqual(
-            [mock.call.mysql_mysqldb(url, 15)],
+            [mock.call.mysql_pymysql(url, 15)],
+            callable_fn.mock_calls
+        )
+
+    def test_url_mysql_generic(self):
+        url = sqlalchemy.engine.url.make_url(
+            "mysql://scott:tiger@localhost/test")
+        dispatcher, callable_fn = self._single_fixture()
+
+        dispatcher(url, 15)
+        self.assertEqual(
+            [mock.call.mysql(url, 15)],
             callable_fn.mock_calls
         )
 
@@ -1030,9 +1048,10 @@ class TestDialectFunctionDispatcher(test_base.BaseTestCase):
         dispatcher, callable_fn = self._multiple_fixture()
 
         dispatcher("postgresql+pyodbc://", 1)
-        dispatcher("mysql://", 2)
+        dispatcher("mysql+pymysql://", 2)
         dispatcher("ibm_db_sa+db2://", 3)
         dispatcher("postgresql+psycopg2://", 4)
+        dispatcher("postgresql://", 5)
 
         # TODO(zzzeek): there is a deterministic order here, but we might
         # want to tweak it, or maybe provide options.  default first?
@@ -1042,12 +1061,18 @@ class TestDialectFunctionDispatcher(test_base.BaseTestCase):
                 mock.call.postgresql('postgresql+pyodbc://', 1),
                 mock.call.pyodbc('postgresql+pyodbc://', 1),
                 mock.call.default('postgresql+pyodbc://', 1),
-                mock.call.mysql_mysqldb('mysql://', 2),
-                mock.call.default('mysql://', 2),
+                mock.call.mysql_pymysql('mysql+pymysql://', 2),
+                mock.call.mysql('mysql+pymysql://', 2),
+                mock.call.default('mysql+pymysql://', 2),
                 mock.call.default('ibm_db_sa+db2://', 3),
                 mock.call.postgresql_psycopg2('postgresql+psycopg2://', 4),
                 mock.call.postgresql('postgresql+psycopg2://', 4),
                 mock.call.default('postgresql+psycopg2://', 4),
+                # note this is called because we resolve the default
+                # DBAPI for the url
+                mock.call.postgresql_psycopg2('postgresql://', 5),
+                mock.call.postgresql('postgresql://', 5),
+                mock.call.default('postgresql://', 5),
             ],
             callable_fn.mock_calls
         )
diff --git a/oslo_db/tests/sqlalchemy/test_exc_filters.py b/oslo_db/tests/sqlalchemy/test_exc_filters.py
index 34313ce..5e93dd4 100644
--- a/oslo_db/tests/sqlalchemy/test_exc_filters.py
+++ b/oslo_db/tests/sqlalchemy/test_exc_filters.py
@@ -46,7 +46,16 @@ class _SQLAExceptionMatcher(object):
             self.assertTrue(issubclass(exc.__class__, exception_type))
         else:
             self.assertEqual(exc.__class__.__name__, exception_type)
-        self.assertEqual(str(exc.orig).lower(), message.lower())
+        if isinstance(message, tuple):
+            self.assertEqual(
+                [a.lower()
+                 if isinstance(a, six.string_types) else a
+                 for a in exc.orig.args],
+                [m.lower()
+                 if isinstance(m, six.string_types) else m for m in message]
+            )
+        else:
+            self.assertEqual(str(exc.orig).lower(), message.lower())
         if sql is not None:
             self.assertEqual(exc.statement, sql)
         if params is not None:
@@ -362,10 +371,10 @@ class TestReferenceErrorMySQL(TestReferenceErrorSQLite,
         self.assertInnerException(
             matched,
             "IntegrityError",
-            "(1452, 'Cannot add or update a child row: a "
-            "foreign key constraint fails (`{0}`.`resource_entity`, "
-            "CONSTRAINT `foo_fkey` FOREIGN KEY (`foo_id`) REFERENCES "
-            "`resource_foo` (`id`))')".format(self.engine.url.database),
+            (1452, "Cannot add or update a child row: a "
+             "foreign key constraint fails (`{0}`.`resource_entity`, "
+             "CONSTRAINT `foo_fkey` FOREIGN KEY (`foo_id`) REFERENCES "
+             "`resource_foo` (`id`))".format(self.engine.url.database)),
             "INSERT INTO resource_entity (id, foo_id) VALUES (%s, %s)",
             (1, 2)
         )
@@ -390,10 +399,13 @@ class TestReferenceErrorMySQL(TestReferenceErrorSQLite,
         self.assertInnerException(
             matched,
             "IntegrityError",
-            '(1452, \'Cannot add or update a child row: a '
-            'foreign key constraint fails ("{0}"."resource_entity", '
-            'CONSTRAINT "foo_fkey" FOREIGN KEY ("foo_id") REFERENCES '
-            '"resource_foo" ("id"))\')'.format(self.engine.url.database),
+            (
+                1452,
+                'Cannot add or update a child row: a '
+                'foreign key constraint fails ("{0}"."resource_entity", '
+                'CONSTRAINT "foo_fkey" FOREIGN KEY ("foo_id") REFERENCES '
+                '"resource_foo" ("id"))'.format(self.engine.url.database)
+            ),
             "INSERT INTO resource_entity (id, foo_id) VALUES (%s, %s)",
             (1, 2)
         )
@@ -414,11 +426,14 @@ class TestReferenceErrorMySQL(TestReferenceErrorSQLite,
         self.assertInnerException(
             matched,
             "IntegrityError",
-            "(1451, 'cannot delete or update a parent row: a foreign key "
-            "constraint fails (`{0}`.`resource_entity`, "
-            "constraint `foo_fkey` "
-            "foreign key (`foo_id`) references "
-            "`resource_foo` (`id`))')".format(self.engine.url.database),
+            (
+                1451,
+                "Cannot delete or update a parent row: a foreign key "
+                "constraint fails (`{0}`.`resource_entity`, "
+                "constraint `foo_fkey` "
+                "foreign key (`foo_id`) references "
+                "`resource_foo` (`id`))".format(self.engine.url.database)
+            ),
             "DELETE FROM resource_foo",
             (),
         )
@@ -483,7 +498,7 @@ class TestDuplicate(TestsExceptionFilter):
             "PRIMARY KEY must be unique 'insert into t values(10)'",
             expected_columns=[])
 
-    def test_mysql_mysqldb(self):
+    def test_mysql_pymysql(self):
         self._run_dupe_constraint_test(
             "mysql",
             '(1062, "Duplicate entry '
@@ -606,7 +621,7 @@ class TestDeadlock(TestsExceptionFilter):
 
         self.assertEqual(matched.orig.__class__.__name__, expected_dbapi_cls)
 
-    def test_mysql_mysqldb_deadlock(self):
+    def test_mysql_pymysql_deadlock(self):
         self._run_deadlock_detect_test(
             "mysql",
             "(1213, 'Deadlock found when trying "
@@ -614,7 +629,7 @@ class TestDeadlock(TestsExceptionFilter):
             "transaction')"
         )
 
-    def test_mysql_mysqldb_galera_deadlock(self):
+    def test_mysql_pymysql_galera_deadlock(self):
         self._run_deadlock_detect_test(
             "mysql",
             "(1205, 'Lock wait timeout exceeded; "
diff --git a/oslo_db/tests/sqlalchemy/test_sqlalchemy.py b/oslo_db/tests/sqlalchemy/test_sqlalchemy.py
index 02b0af5..671cd27 100644
--- a/oslo_db/tests/sqlalchemy/test_sqlalchemy.py
+++ b/oslo_db/tests/sqlalchemy/test_sqlalchemy.py
@@ -558,7 +558,7 @@ class CreateEngineTest(oslo_test.BaseTestCase):
 
     def test_queuepool_args(self):
         engines._init_connection_args(
-            url.make_url("mysql://u:p@host/test"), self.args,
+            url.make_url("mysql+pymysql://u:p@host/test"), self.args,
             max_pool_size=10, max_overflow=10)
         self.assertEqual(self.args['pool_size'], 10)
         self.assertEqual(self.args['max_overflow'], 10)
@@ -609,6 +609,12 @@ class CreateEngineTest(oslo_test.BaseTestCase):
         self.assertEqual(self.args['connect_args'],
                          {'charset': 'utf8', 'use_unicode': 0})
 
+    def test_mysql_pymysql_connect_args_default(self):
+        engines._init_connection_args(
+            url.make_url("mysql+pymysql://u:p@host/test"), self.args)
+        self.assertEqual(self.args['connect_args'],
+                         {'charset': 'utf8'})
+
     def test_mysql_mysqldb_connect_args_default(self):
         engines._init_connection_args(
             url.make_url("mysql+mysqldb://u:p@host/test"), self.args)
diff --git a/oslo_db/tests/sqlalchemy/test_utils.py b/oslo_db/tests/sqlalchemy/test_utils.py
index 6719d0f..054f99d 100644
--- a/oslo_db/tests/sqlalchemy/test_utils.py
+++ b/oslo_db/tests/sqlalchemy/test_utils.py
@@ -912,8 +912,10 @@ class TestDialectFunctionDispatcher(test_base.BaseTestCase):
         dispatcher = orig = utils.dispatch_for_dialect("*")(
             callable_fn.default)
         dispatcher = dispatcher.dispatch_for("sqlite")(callable_fn.sqlite)
-        dispatcher = dispatcher.dispatch_for("mysql+mysqldb")(
-            callable_fn.mysql_mysqldb)
+        dispatcher = dispatcher.dispatch_for("mysql+pymysql")(
+            callable_fn.mysql_pymysql)
+        dispatcher = dispatcher.dispatch_for("mysql")(
+            callable_fn.mysql)
         dispatcher = dispatcher.dispatch_for("postgresql")(
             callable_fn.postgresql)
 
@@ -927,7 +929,8 @@ class TestDialectFunctionDispatcher(test_base.BaseTestCase):
         for targ in [
             callable_fn.default,
             callable_fn.sqlite,
-            callable_fn.mysql_mysqldb,
+            callable_fn.mysql,
+            callable_fn.mysql_pymysql,
             callable_fn.postgresql,
             callable_fn.postgresql_psycopg2,
             callable_fn.pyodbc
@@ -937,8 +940,10 @@ class TestDialectFunctionDispatcher(test_base.BaseTestCase):
         dispatcher = orig = utils.dispatch_for_dialect("*", multiple=True)(
             callable_fn.default)
         dispatcher = dispatcher.dispatch_for("sqlite")(callable_fn.sqlite)
-        dispatcher = dispatcher.dispatch_for("mysql+mysqldb")(
-            callable_fn.mysql_mysqldb)
+        dispatcher = dispatcher.dispatch_for("mysql+pymysql")(
+            callable_fn.mysql_pymysql)
+        dispatcher = dispatcher.dispatch_for("mysql")(
+            callable_fn.mysql)
         dispatcher = dispatcher.dispatch_for("postgresql+*")(
             callable_fn.postgresql)
         dispatcher = dispatcher.dispatch_for("postgresql+psycopg2")(
@@ -955,15 +960,17 @@ class TestDialectFunctionDispatcher(test_base.BaseTestCase):
         dispatcher, callable_fn = self._single_fixture()
         dispatcher("sqlite://", 1)
         dispatcher("postgresql+psycopg2://u:p@h/t", 2)
-        dispatcher("mysql://u:p@h/t", 3)
-        dispatcher("mysql+mysqlconnector://u:p@h/t", 4)
+        dispatcher("mysql+pymysql://u:p@h/t", 3)
+        dispatcher("mysql://u:p@h/t", 4)
+        dispatcher("mysql+mysqlconnector://u:p@h/t", 5)
 
         self.assertEqual(
             [
                 mock.call.sqlite('sqlite://', 1),
                 mock.call.postgresql("postgresql+psycopg2://u:p@h/t", 2),
-                mock.call.mysql_mysqldb("mysql://u:p@h/t", 3),
-                mock.call.default("mysql+mysqlconnector://u:p@h/t", 4)
+                mock.call.mysql_pymysql("mysql+pymysql://u:p@h/t", 3),
+                mock.call.mysql("mysql://u:p@h/t", 4),
+                mock.call.mysql("mysql+mysqlconnector://u:p@h/t", 5),
             ],
             callable_fn.mock_calls)
 
@@ -1081,10 +1088,10 @@ class TestDialectFunctionDispatcher(test_base.BaseTestCase):
 
     def test_single_retval(self):
         dispatcher, callable_fn = self._single_fixture()
-        callable_fn.mysql_mysqldb.return_value = 5
+        callable_fn.mysql_pymysql.return_value = 5
 
         self.assertEqual(
-            dispatcher("mysql://u:p@h/t", 3), 5
+            dispatcher("mysql+pymysql://u:p@h/t", 3), 5
         )
 
     def test_engine(self):
@@ -1097,14 +1104,25 @@ class TestDialectFunctionDispatcher(test_base.BaseTestCase):
             callable_fn.mock_calls
         )
 
-    def test_url(self):
+    def test_url_pymysql(self):
         url = sqlalchemy.engine.url.make_url(
-            "mysql+mysqldb://scott:tiger@localhost/test")
+            "mysql+pymysql://scott:tiger@localhost/test")
         dispatcher, callable_fn = self._single_fixture()
 
         dispatcher(url, 15)
         self.assertEqual(
-            [mock.call.mysql_mysqldb(url, 15)],
+            [mock.call.mysql_pymysql(url, 15)],
+            callable_fn.mock_calls
+        )
+
+    def test_url_mysql_generic(self):
+        url = sqlalchemy.engine.url.make_url(
+            "mysql://scott:tiger@localhost/test")
+        dispatcher, callable_fn = self._single_fixture()
+
+        dispatcher(url, 15)
+        self.assertEqual(
+            [mock.call.mysql(url, 15)],
             callable_fn.mock_calls
         )
 
@@ -1149,9 +1167,10 @@ class TestDialectFunctionDispatcher(test_base.BaseTestCase):
         dispatcher, callable_fn = self._multiple_fixture()
 
         dispatcher("postgresql+pyodbc://", 1)
-        dispatcher("mysql://", 2)
+        dispatcher("mysql+pymysql://", 2)
         dispatcher("ibm_db_sa+db2://", 3)
         dispatcher("postgresql+psycopg2://", 4)
+        dispatcher("postgresql://", 5)
 
         # TODO(zzzeek): there is a deterministic order here, but we might
         # want to tweak it, or maybe provide options.  default first?
@@ -1161,12 +1180,18 @@ class TestDialectFunctionDispatcher(test_base.BaseTestCase):
                 mock.call.postgresql('postgresql+pyodbc://', 1),
                 mock.call.pyodbc('postgresql+pyodbc://', 1),
                 mock.call.default('postgresql+pyodbc://', 1),
-                mock.call.mysql_mysqldb('mysql://', 2),
-                mock.call.default('mysql://', 2),
+                mock.call.mysql_pymysql('mysql+pymysql://', 2),
+                mock.call.mysql('mysql+pymysql://', 2),
+                mock.call.default('mysql+pymysql://', 2),
                 mock.call.default('ibm_db_sa+db2://', 3),
                 mock.call.postgresql_psycopg2('postgresql+psycopg2://', 4),
                 mock.call.postgresql('postgresql+psycopg2://', 4),
                 mock.call.default('postgresql+psycopg2://', 4),
+                # note this is called because we resolve the default
+                # DBAPI for the url
+                mock.call.postgresql_psycopg2('postgresql://', 5),
+                mock.call.postgresql('postgresql://', 5),
+                mock.call.default('postgresql://', 5),
             ],
             callable_fn.mock_calls
         )
diff --git a/test-requirements-py2.txt b/test-requirements-py2.txt
deleted file mode 100644
index 93f2273..0000000
--- a/test-requirements-py2.txt
+++ /dev/null
@@ -1,19 +0,0 @@
-# The order of packages is significant, because pip processes them in the order
-# of appearance. Changing the order has an impact on the overall integration
-# process, which may cause wedges in the gate later.
-
-hacking>=0.10.0,<0.11
-
-coverage>=3.6
-discover
-doc8  # Apache-2.0
-fixtures>=0.3.14
-MySQL-python
-psycopg2
-python-subunit>=0.0.18
-sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3
-oslosphinx>=2.5.0  # Apache-2.0
-oslotest>=1.5.1  # Apache-2.0
-testrepository>=0.0.18
-testtools>=0.9.36,!=1.2.0
-tempest-lib>=0.5.0
diff --git a/test-requirements-py3.txt b/test-requirements.txt
index 35c60e6..320463f 100644
--- a/test-requirements-py3.txt
+++ b/test-requirements.txt
@@ -8,12 +8,12 @@ coverage>=3.6
 discover
 doc8  # Apache-2.0
 fixtures>=0.3.14
+PyMySQL>=0.6.2  # MIT License
 psycopg2
 python-subunit>=0.0.18
 sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3
 oslosphinx>=2.5.0  # Apache-2.0
 oslotest>=1.5.1  # Apache-2.0
-PyMySQL>=0.6.2  # MIT License
 testrepository>=0.0.18
 testtools>=0.9.36,!=1.2.0
 tempest-lib>=0.5.0
diff --git a/tox.ini b/tox.ini
index 97b57f4..95b682b 100644
--- a/tox.ini
+++ b/tox.ini
@@ -14,7 +14,7 @@ install_command = pip install -U {opts} {packages}
 setenv =
    VIRTUAL_ENV={envdir}
 deps = -r{toxinidir}/requirements.txt
-       -r{toxinidir}/test-requirements-py2.txt
+       -r{toxinidir}/test-requirements.txt
 commands = bash tools/pretty_tox.sh '{posargs}'
 
 [testenv:sqla_09]
@@ -25,12 +25,12 @@ commands = pip install SQLAlchemy>=0.9.0,!=0.9.5,<1.0.0
 commands = pip install SQLAlchemy>=0.8.0,<0.9.0
             python setup.py testr --slowest --testr-args='{posargs}'
 
-[testenv:py34]
+[testenv:mysql-python]
 setenv =
    {[testenv]setenv}
-   OS_TEST_DBAPI_ADMIN_CONNECTION=mysql+pymysql://openstack_citest:openstack_citest@localhost/;postgresql://openstack_citest:openstack_citest@localhost/postgres;sqlite://
-deps = -r{toxinidir}/requirements.txt
-       -r{toxinidir}/test-requirements-py3.txt
+   OS_TEST_DBAPI_ADMIN_CONNECTION=mysql://openstack_citest:openstack_citest@localhost/;postgresql://openstack_citest:openstack_citest@localhost/postgres;sqlite://
+commands = pip install MySQL-python
+           python setup.py testr --slowest --testr-args='{posargs}'
 
 [testenv:pep8]
 commands = flake8