From 0ea1ee087a35a43e04fcdbee2405e55dc5ef20a4 Mon Sep 17 00:00:00 2001
From: Mitya_Eremeev <mitossvyaz@mail.ru>
Date: Fri, 7 Oct 2022 14:55:14 +0300
Subject: Retry to detach volume if nova returned error 409

During volume detachment an instance can be in error state or locked.
Nova returns error 409.
Instance can be unlocked or fixed a bit later, but heat doesn't retry to detach volume.
The patch makes heat to retry detachments untill stack update timeout.

story: 2010355
Change-Id: I9a08b938401cd71eddd0eb80782d10392f92bf45
---
 heat/engine/resources/volume_base.py       | 13 ++++++++-----
 heat/tests/openstack/cinder/test_volume.py | 23 +++++++++++++++++++++++
 2 files changed, 31 insertions(+), 5 deletions(-)

diff --git a/heat/engine/resources/volume_base.py b/heat/engine/resources/volume_base.py
index f959ce6b7..e870afecc 100644
--- a/heat/engine/resources/volume_base.py
+++ b/heat/engine/resources/volume_base.py
@@ -197,11 +197,10 @@ class BaseVolumeAttachment(resource.Resource):
         if self.resource_id:
             server_id = self.properties[self.INSTANCE_ID]
             vol_id = self.properties[self.VOLUME_ID]
-            self.client_plugin('nova').detach_volume(server_id,
-                                                     self.resource_id)
-            prg = progress.VolumeDetachProgress(
-                server_id, vol_id, self.resource_id)
-            prg.called = True
+            prg = progress.VolumeDetachProgress(server_id, vol_id,
+                                                self.resource_id)
+            prg.called = self.client_plugin('nova').detach_volume(
+                server_id, self.resource_id)
 
         return prg
 
@@ -209,6 +208,10 @@ class BaseVolumeAttachment(resource.Resource):
         if prg is None:
             return True
 
+        if not prg.called:
+            prg.called = self.client_plugin('nova').detach_volume(
+                prg.srv_id, self.resource_id)
+            return False
         if not prg.cinder_complete:
             prg.cinder_complete = self.client_plugin(
             ).check_detach_volume_complete(prg.vol_id, prg.srv_id)
diff --git a/heat/tests/openstack/cinder/test_volume.py b/heat/tests/openstack/cinder/test_volume.py
index 56c28b98b..a137b226a 100644
--- a/heat/tests/openstack/cinder/test_volume.py
+++ b/heat/tests/openstack/cinder/test_volume.py
@@ -1404,3 +1404,26 @@ class CinderVolumeTest(vt_base.VolumeTestCase):
         self.assertEqual(True, rsrc._ready_to_extend_volume())
         self.assertEqual(False, rsrc._ready_to_extend_volume())
         self.assertEqual(True, rsrc._ready_to_extend_volume())
+
+    def test_try_detach_volume_if_server_was_temporarily_in_error(self):
+        self.stack_name = 'test_cvolume_detach_server_in_error_stack'
+        fva = vt_base.FakeVolume('in-use')
+        m_v = self._mock_create_server_volume_script(
+            vt_base.FakeVolume('attaching'))
+        self._mock_create_volume(vt_base.FakeVolume('creating'),
+                                 self.stack_name,
+                                 extra_get_mocks=[
+                                     m_v, fva,
+                                     vt_base.FakeVolume('available')])
+        self.stub_VolumeConstraint_validate()
+        # delete script
+        self.fc.volumes.get_server_volume.side_effect = [
+            fva, fva, fakes_nova.fake_exception()]
+        nova_responses = [nova_exp.Conflict('409'), None]
+        self.fc.volumes.delete_server_volume.side_effect = nova_responses
+        stack = utils.parse_stack(self.t, stack_name=self.stack_name)
+        self.create_volume(self.t, stack, 'volume')
+        rsrc = self.create_attachment(self.t, stack, 'attachment')
+        scheduler.TaskRunner(rsrc.delete)()
+        self.assertEqual(self.fc.volumes.delete_server_volume.call_count,
+                         len(nova_responses))
-- 
cgit v1.2.1