Add retry to cinder API calls related to volume detach

When shutting down an instance for which volume needs to be
deleted, if cinder RPC timeout expires before cinder volume
driver terminates connection, then an unknown cinder exception
is received and the volume is not removed.

This fix adds a retry mechanism directly in cinder API calls
attachment_delete, terminate_connection, and detach.

Change-Id: I3c9ae47d0ceb64fa3082a01cb7df27faa4f5a00d
Closes-Bug: #1834659

1 files changed, 11 insertions, 0 deletions

diff --git a/nova/volume/cinder.py b/nova/volume/cinder.py
index addf58ec77..5940c16891 100644
--- a/nova/volume/cinder.py
+++ b/nova/volume/cinder.py
@@ -24,6 +24,7 @@ import functools
 import sys
 
 from cinderclient import api_versions as cinder_api_versions
+from cinderclient import apiclient as cinder_apiclient
 from cinderclient import client as cinder_client
 from cinderclient import exceptions as cinder_exception
 from keystoneauth1 import exceptions as keystone_exception
@@ -33,6 +34,7 @@ from oslo_serialization import jsonutils
 from oslo_utils import encodeutils
 from oslo_utils import excutils
 from oslo_utils import strutils
+import retrying
 import six
 from six.moves import urllib
 
@@ -545,6 +547,9 @@ class API(object):
                                              mountpoint, mode=mode)
 
     @translate_volume_exception
+    @retrying.retry(stop_max_attempt_number=5,
+                    retry_on_exception=lambda e:
+                    type(e) == cinder_apiclient.exceptions.InternalServerError)
     def detach(self, context, volume_id, instance_uuid=None,
                attachment_id=None):
         client = cinderclient(context)
@@ -608,6 +613,9 @@ class API(object):
                                 exc.code if hasattr(exc, 'code') else None)})
 
     @translate_volume_exception
+    @retrying.retry(stop_max_attempt_number=5,
+                    retry_on_exception=lambda e:
+                    type(e) == cinder_apiclient.exceptions.InternalServerError)
     def terminate_connection(self, context, volume_id, connector):
         return cinderclient(context).volumes.terminate_connection(volume_id,
                                                                   connector)
@@ -854,6 +862,9 @@ class API(object):
                            'code': getattr(ex, 'code', None)})
 
     @translate_attachment_exception
+    @retrying.retry(stop_max_attempt_number=5,
+                    retry_on_exception=lambda e:
+                    type(e) == cinder_apiclient.exceptions.InternalServerError)
     def attachment_delete(self, context, attachment_id):
         try:
             cinderclient(