Handle DB failures in servicegroup DB driver

Fix an issue where when local conductor is used, the DB driver for
servicegroup will not handle transient DB problems gracefully.  The
patch makes the behavior consistent with messaging timeouts if remote
conductor is used.

Change-Id: Ie736e7b64ea4f60f78878c8713ce826702f9f05e
Closes-Bug: 1505471

1 files changed, 12 insertions, 3 deletions

diff --git a/nova/servicegroup/drivers/db.py b/nova/servicegroup/drivers/db.py
index ed9106e732..28471d86f1 100644
--- a/nova/servicegroup/drivers/db.py
+++ b/nova/servicegroup/drivers/db.py
@@ -14,6 +14,7 @@
 # limitations under the License.
 
 from oslo_config import cfg
+from oslo_db import exception as db_exception
 from oslo_log import log as logging
 import oslo_messaging as messaging
 from oslo_utils import timeutils
@@ -26,6 +27,7 @@ from nova.servicegroup.drivers import base
 
 CONF = cfg.CONF
 CONF.import_opt('service_down_time', 'nova.service')
+CONF.import_opt('use_local', 'nova.conductor.api', group='conductor')
 
 LOG = logging.getLogger(__name__)
 
@@ -82,6 +84,14 @@ class DbDriver(base.Driver):
 
     def _report_state(self, service):
         """Update the state of this service in the datastore."""
+
+        if CONF.conductor.use_local:
+            # need to catch DB type errors
+            exc_cls = db_exception.DBError  # oslo.db exception base class
+        else:
+            # need to catch messaging timeouts
+            exc_cls = messaging.MessagingTimeout
+
         try:
             service.service_ref.report_count += 1
             service.service_ref.save()
@@ -93,9 +103,8 @@ class DbDriver(base.Driver):
                     _LI('Recovered connection to nova-conductor '
                         'for reporting service status.'))
 
-        # because we are communicating over conductor, a failure to
-        # connect is going to be a messaging failure, not a db error.
-        except messaging.MessagingTimeout:
+        # the type of failure depends on use of remote or local conductor
+        except exc_cls:
             if not getattr(service, 'model_disconnected', False):
                 service.model_disconnected = True
                 LOG.warn(_LW('Lost connection to nova-conductor '