Merge "Workaround client race in legacy nested stack delete"

5 files changed, 52 insertions, 6 deletions

diff --git a/heat/engine/resources/stack_resource.py b/heat/engine/resources/stack_resource.py
index ce6ec78f5..5b491888a 100644
--- a/heat/engine/resources/stack_resource.py
+++ b/heat/engine/resources/stack_resource.py
@@ -425,6 +425,11 @@ class StackResource(resource.Resource):
             return False
 
         if status == self.IN_PROGRESS:
+            if cookie is not None and 'fail_count' in cookie:
+                prev_status_reason = cookie['previous']['status_reason']
+                if status_reason != prev_status_reason:
+                    # State has changed, so fail on the next failure
+                    cookie['fail_count'] = 1
             return False
         elif status == self.COMPLETE:
             # For operations where we do not take a resource lock
@@ -438,6 +443,10 @@ class StackResource(resource.Resource):
                 self._nested = None
             return done
         elif status == self.FAILED:
+            if cookie is not None and 'fail_count' in cookie:
+                cookie['fail_count'] -= 1
+                if cookie['fail_count'] > 0:
+                    raise resource.PollDelay(10)
             raise exception.ResourceFailure(status_reason, self,
                                             action=action)
         else:
@@ -564,12 +573,33 @@ class StackResource(resource.Resource):
         if stack_identity is None:
             return
 
+        cookie = None
+        if not self.stack.convergence:
+            try:
+                status_data = stack_object.Stack.get_status(self.context,
+                                                            self.resource_id)
+            except exception.NotFound:
+                return
+
+            action, status, status_reason, updated_time = status_data
+            if (action, status) == (self.stack.DELETE,
+                                    self.stack.IN_PROGRESS):
+                cookie = {
+                    'previous': {
+                        'state': (action, status),
+                        'status_reason': status_reason,
+                        'updated_at': None,
+                    },
+                    'fail_count': 2,
+                }
+
         with self.rpc_client().ignore_error_by_name('EntityNotFound'):
             if self.abandon_in_progress:
                 self.rpc_client().abandon_stack(self.context, stack_identity)
             else:
                 self.rpc_client().delete_stack(self.context, stack_identity,
                                                cast=False)
+            return cookie
 
     def handle_delete(self):
         return self.delete_nested()
diff --git a/heat/engine/stack.py b/heat/engine/stack.py
index aa65b92e8..a01a76591 100644
--- a/heat/engine/stack.py
+++ b/heat/engine/stack.py
@@ -1983,8 +1983,8 @@ class Stack(collections.Mapping):
 
         stack_status = self.COMPLETE
         reason = 'Stack %s completed successfully' % action
-        self.state_set(action, self.IN_PROGRESS, 'Stack %s started' %
-                       action)
+        self.state_set(action, self.IN_PROGRESS, 'Stack %s started at %s' %
+                       (action, oslo_timeutils.utcnow().isoformat()))
         if notify is not None:
             notify.signal()
 
diff --git a/heat/tests/test_nested_stack.py b/heat/tests/test_nested_stack.py
index 2fd71eecf..33a9ab9fd 100644
--- a/heat/tests/test_nested_stack.py
+++ b/heat/tests/test_nested_stack.py
@@ -30,6 +30,7 @@ from heat.engine import rsrc_defn
 from heat.engine import stack as parser
 from heat.engine import template
 from heat.objects import resource_data as resource_data_object
+from heat.objects import stack as stack_object
 from heat.tests import common
 from heat.tests import generic_resource as generic_rsrc
 from heat.tests import utils
@@ -387,6 +388,10 @@ Outputs:
         self.assertIsNone(self.res.validate())
         self.res.store()
 
+        self.patchobject(stack_object.Stack, 'get_status',
+                         return_value=('CREATE', 'COMPLETE',
+                                       'Created', 'Sometime'))
+
     def test_handle_create(self):
         self.res.create_with_template = mock.Mock(return_value=None)
 
diff --git a/heat/tests/test_provider_template.py b/heat/tests/test_provider_template.py
index 360a1c3e6..9064b51ed 100644
--- a/heat/tests/test_provider_template.py
+++ b/heat/tests/test_provider_template.py
@@ -32,6 +32,7 @@ from heat.engine import rsrc_defn
 from heat.engine import stack as parser
 from heat.engine import support
 from heat.engine import template
+from heat.objects import stack as stack_object
 from heat.tests import common
 from heat.tests import generic_resource as generic_rsrc
 from heat.tests import utils
@@ -972,6 +973,10 @@ class TemplateResourceCrudTest(common.HeatTestCase):
                                                       self.defn, self.stack)
         self.assertIsNone(self.res.validate())
 
+        self.patchobject(stack_object.Stack, 'get_status',
+                         return_value=('CREATE', 'COMPLETE',
+                                       'Created', 'Sometime'))
+
     def test_handle_create(self):
         self.res.create_with_template = mock.Mock(return_value=None)
 
diff --git a/heat/tests/test_stack_resource.py b/heat/tests/test_stack_resource.py
index 76084a566..1d632f650 100644
--- a/heat/tests/test_stack_resource.py
+++ b/heat/tests/test_stack_resource.py
@@ -219,7 +219,10 @@ class StackResourceTest(StackResourceBaseTest):
         self.parent_resource.resource_id = 'fake_id'
 
         self.parent_resource.prepare_abandon()
-        self.parent_resource.delete_nested()
+        status = ('CREATE', 'COMPLETE', '', 'now_time')
+        with mock.patch.object(stack_object.Stack, 'get_status',
+                               return_value=status):
+            self.parent_resource.delete_nested()
 
         rpcc.return_value.abandon_stack.assert_called_once_with(
             self.parent_resource.context, mock.ANY)
@@ -522,13 +525,16 @@ class StackResourceTest(StackResourceBaseTest):
         def exc_filter(*args):
             try:
                 yield
-            except exception.NotFound:
+            except exception.EntityNotFound:
                 pass
 
         rpcc.return_value.ignore_error_by_name.side_effect = exc_filter
         rpcc.return_value.delete_stack = mock.Mock(
-            side_effect=exception.NotFound())
-        self.assertIsNone(self.parent_resource.delete_nested())
+            side_effect=exception.EntityNotFound('Stack', 'nested'))
+        status = ('CREATE', 'COMPLETE', '', 'now_time')
+        with mock.patch.object(stack_object.Stack, 'get_status',
+                               return_value=status):
+            self.assertIsNone(self.parent_resource.delete_nested())
         rpcc.return_value.delete_stack.assert_called_once_with(
             self.parent_resource.context, mock.ANY, cast=False)