Dispose db connections pool on disconnect

When a disconnect happens (e.g. when VIP moves to another node), all
existing connections in the pool become invalid. There is no sense to
check every single connection in the pool, as this may take a long
time (e.g. when python-mysqldb driver is used with eventlet green
threads, there is no context switch on db IO, so the whole process is
blocked until socket read times out). Dispose all connections in the
pool when disconnect is detected, they will be recreated on demand
then.

Closes-Bug: #1288438

Change-Id: Ia357da2b3092d306a86b6d1787bab374a335f28c

2 files changed, 17 insertions, 5 deletions

diff --git a/openstack/common/db/sqlalchemy/session.py b/openstack/common/db/sqlalchemy/session.py
index ebe72ff..ba42616 100644
--- a/openstack/common/db/sqlalchemy/session.py
+++ b/openstack/common/db/sqlalchemy/session.py
@@ -505,6 +505,14 @@ def _ping_listener(engine, dbapi_conn, connection_rec, connection_proxy):
         if engine.dialect.is_disconnect(ex, dbapi_conn, cursor):
             msg = _LW('Database server has gone away: %s') % ex
             LOG.warning(msg)
+
+            # if the database server has gone away, all connections in the pool
+            # have become invalid and we can safely close all of them here,
+            # rather than waste time on checking of every single connection
+            engine.dispose()
+
+            # this will be handled by SQLAlchemy and will force it to create
+            # a new connection and retry the original action
             raise sqla_exc.DisconnectionError(msg)
         else:
             raise
diff --git a/tests/unit/db/sqlalchemy/test_sqlalchemy.py b/tests/unit/db/sqlalchemy/test_sqlalchemy.py
index c9ae638..49a1162 100644
--- a/tests/unit/db/sqlalchemy/test_sqlalchemy.py
+++ b/tests/unit/db/sqlalchemy/test_sqlalchemy.py
@@ -209,6 +209,9 @@ class FakeDB2Engine(object):
     dialect = Dialect()
     name = 'ibm_db_sa'
 
+    def dispose(self):
+        pass
+
 
 class TestDBDisconnected(test.BaseTestCase):
 
@@ -219,11 +222,12 @@ class TestDBDisconnected(test.BaseTestCase):
             'convert_unicode': True}
 
         engine = sqlalchemy.create_engine(connection, **engine_args)
-
-        self.assertRaises(sqlalchemy.exc.DisconnectionError,
-                          session._ping_listener, engine,
-                          FakeDBAPIConnection(), FakeConnectionRec(),
-                          FakeConnectionProxy())
+        with mock.patch.object(engine, 'dispose') as dispose_mock:
+            self.assertRaises(sqlalchemy.exc.DisconnectionError,
+                              session._ping_listener, engine,
+                              FakeDBAPIConnection(), FakeConnectionRec(),
+                              FakeConnectionProxy())
+            dispose_mock.assert_called_once_with()
 
     def test_mysql_ping_listener_disconnected(self):
         def fake_execute(sql):