Retry to detach volume if nova returned error 409

During volume detachment an instance can be in error state or locked.
Nova returns error 409.
Instance can be unlocked or fixed a bit later, but heat doesn't retry to detach volume.
The patch makes heat to retry detachments untill stack update timeout.

story: 2010355
Change-Id: I9a08b938401cd71eddd0eb80782d10392f92bf45

2 files changed, 31 insertions, 5 deletions

diff --git a/heat/engine/resources/volume_base.py b/heat/engine/resources/volume_base.py
index f959ce6b7..e870afecc 100644
--- a/heat/engine/resources/volume_base.py
+++ b/heat/engine/resources/volume_base.py
@@ -197,11 +197,10 @@ class BaseVolumeAttachment(resource.Resource):
         if self.resource_id:
             server_id = self.properties[self.INSTANCE_ID]
             vol_id = self.properties[self.VOLUME_ID]
-            self.client_plugin('nova').detach_volume(server_id,
-                                                     self.resource_id)
-            prg = progress.VolumeDetachProgress(
-                server_id, vol_id, self.resource_id)
-            prg.called = True
+            prg = progress.VolumeDetachProgress(server_id, vol_id,
+                                                self.resource_id)
+            prg.called = self.client_plugin('nova').detach_volume(
+                server_id, self.resource_id)
 
         return prg
 
@@ -209,6 +208,10 @@ class BaseVolumeAttachment(resource.Resource):
         if prg is None:
             return True
 
+        if not prg.called:
+            prg.called = self.client_plugin('nova').detach_volume(
+                prg.srv_id, self.resource_id)
+            return False
         if not prg.cinder_complete:
             prg.cinder_complete = self.client_plugin(
             ).check_detach_volume_complete(prg.vol_id, prg.srv_id)
diff --git a/heat/tests/openstack/cinder/test_volume.py b/heat/tests/openstack/cinder/test_volume.py
index 56c28b98b..a137b226a 100644
--- a/heat/tests/openstack/cinder/test_volume.py
+++ b/heat/tests/openstack/cinder/test_volume.py
@@ -1404,3 +1404,26 @@ class CinderVolumeTest(vt_base.VolumeTestCase):
         self.assertEqual(True, rsrc._ready_to_extend_volume())
         self.assertEqual(False, rsrc._ready_to_extend_volume())
         self.assertEqual(True, rsrc._ready_to_extend_volume())
+
+    def test_try_detach_volume_if_server_was_temporarily_in_error(self):
+        self.stack_name = 'test_cvolume_detach_server_in_error_stack'
+        fva = vt_base.FakeVolume('in-use')
+        m_v = self._mock_create_server_volume_script(
+            vt_base.FakeVolume('attaching'))
+        self._mock_create_volume(vt_base.FakeVolume('creating'),
+                                 self.stack_name,
+                                 extra_get_mocks=[
+                                     m_v, fva,
+                                     vt_base.FakeVolume('available')])
+        self.stub_VolumeConstraint_validate()
+        # delete script
+        self.fc.volumes.get_server_volume.side_effect = [
+            fva, fva, fakes_nova.fake_exception()]
+        nova_responses = [nova_exp.Conflict('409'), None]
+        self.fc.volumes.delete_server_volume.side_effect = nova_responses
+        stack = utils.parse_stack(self.t, stack_name=self.stack_name)
+        self.create_volume(self.t, stack, 'volume')
+        rsrc = self.create_attachment(self.t, stack, 'attachment')
+        scheduler.TaskRunner(rsrc.delete)()
+        self.assertEqual(self.fc.volumes.delete_server_volume.call_count,
+                         len(nova_responses))