Retry connections to Nova

Sometimes Neutron is failing to send notification to Nova
due to timeout, refused connection or another HTTP error.
Retry send in those cases.

Closes-Bug: #1987780
Change-Id: Iaaccec770484234b704f70f3c144efac4d8ffba0

1 files changed, 12 insertions, 0 deletions

diff --git a/neutron/notifiers/nova.py b/neutron/notifiers/nova.py
index e48f675d37..1269187ec9 100644
--- a/neutron/notifiers/nova.py
+++ b/neutron/notifiers/nova.py
@@ -16,6 +16,7 @@
 import contextlib
 
 from eventlet.green import threading
+from keystoneauth1 import exceptions as ks_exceptions
 from keystoneauth1 import loading as ks_loading
 from neutron_lib.callbacks import events
 from neutron_lib.callbacks import registry
@@ -32,6 +33,7 @@ from oslo_context import context as common_context
 from oslo_log import log as logging
 from oslo_utils import uuidutils
 from sqlalchemy.orm import attributes as sql_attr
+import tenacity
 
 from neutron.notifiers import batch_notifier
 
@@ -267,6 +269,12 @@ class Notifier(object):
              'tag': port.id})
         self.send_port_status(None, None, port)
 
+    @tenacity.retry(
+        retry=tenacity.retry_if_exception_type(
+            ks_exceptions.RetriableConnectionFailure),
+        wait=tenacity.wait_exponential(multiplier=0.01, max=1),
+        stop=tenacity.stop_after_delay(1),
+        after=tenacity.after_log(LOG, logging.DEBUG))
     def send_events(self, batched_events):
         LOG.debug("Sending events: %s", batched_events)
         novaclient = self._get_nova_client()
@@ -276,6 +284,10 @@ class Notifier(object):
         except nova_exceptions.NotFound:
             LOG.debug("Nova returned NotFound for event: %s",
                       batched_events)
+        except ks_exceptions.RetriableConnectionFailure:
+            raise
+            # next clause handles all exceptions
+            # so reraise for retry decorator
         except Exception:
             LOG.exception("Failed to notify nova on events: %s",
                           batched_events)