Retry connections to Nova

Sometimes Neutron is failing to send notification to Nova
due to timeout, refused connection or another HTTP error.
Retry send in those cases.

Conflicts:
    neutron/notifiers/nova.py
    neutron/tests/unit/notifiers/test_nova.py

Closes-Bug: #1987780
Change-Id: Iaaccec770484234b704f70f3c144efac4d8ffba0
(cherry picked from commit cd475f9af898b81d98b3e0d3f55b94ea653c193c)
(cherry picked from commit b5e9148cc7311080ba1b2a410145949c1adaa0ca)
(cherry picked from commit 49f49bc2bf3b477df00a92e25a8de66221ec8ee6)

2 files changed, 23 insertions, 0 deletions

diff --git a/neutron/notifiers/nova.py b/neutron/notifiers/nova.py
index 3331f7d5f0..7a68e29b41 100644
--- a/neutron/notifiers/nova.py
+++ b/neutron/notifiers/nova.py
@@ -13,6 +13,8 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+
+from keystoneauth1 import exceptions as ks_exceptions
 from keystoneauth1 import loading as ks_loading
 from neutron_lib.callbacks import events
 from neutron_lib.callbacks import registry
@@ -29,6 +31,7 @@ from oslo_context import context as common_context
 from oslo_log import log as logging
 from oslo_utils import uuidutils
 from sqlalchemy.orm import attributes as sql_attr
+import tenacity
 
 from neutron.notifiers import batch_notifier
 
@@ -242,6 +245,12 @@ class Notifier(object):
              'tag': port.id})
         self.send_port_status(None, None, port)
 
+    @tenacity.retry(
+        retry=tenacity.retry_if_exception_type(
+            ks_exceptions.RetriableConnectionFailure),
+        wait=tenacity.wait_exponential(multiplier=0.01, max=1),
+        stop=tenacity.stop_after_delay(1),
+        after=tenacity.after_log(LOG, logging.DEBUG))
     def send_events(self, batched_events):
         LOG.debug("Sending events: %s", batched_events)
         novaclient = self._get_nova_client()
@@ -251,6 +260,10 @@ class Notifier(object):
         except nova_exceptions.NotFound:
             LOG.debug("Nova returned NotFound for event: %s",
                       batched_events)
+        except ks_exceptions.RetriableConnectionFailure:
+            raise
+            # next clause handles all exceptions
+            # so reraise for retry decorator
         except Exception:
             LOG.exception("Failed to notify nova on events: %s",
                           batched_events)
diff --git a/neutron/tests/unit/notifiers/test_nova.py b/neutron/tests/unit/notifiers/test_nova.py
index 4b4607e340..a1a76b8fd6 100644
--- a/neutron/tests/unit/notifiers/test_nova.py
+++ b/neutron/tests/unit/notifiers/test_nova.py
@@ -15,6 +15,8 @@
 
 
 import mock
+
+from keystoneauth1 import exceptions as ks_exc
 from neutron_lib import constants as n_const
 from neutron_lib import context as n_ctx
 from neutron_lib import exceptions as n_exc
@@ -247,6 +249,14 @@ class TestNovaNotify(base.BaseTestCase):
         self.nova_notifier.send_events([])
 
     @mock.patch('novaclient.client.Client')
+    def test_nova_send_events_raises_connect_exc(self, mock_client):
+        create = mock_client().server_external_events.create
+        create.side_effect = (
+            ks_exc.ConnectFailure, ks_exc.ConnectTimeout, [])
+        self.nova_notifier.send_events([])
+        self.assertEqual(3, create.call_count)
+
+    @mock.patch('novaclient.client.Client')
     def test_nova_send_events_raises(self, mock_client):
         mock_client.server_external_events.create.return_value = Exception
         self.nova_notifier.send_events([])