Sleep between check_resource calls

Instead of calling check_resource on all leaves in the resource graph at
once, sleep a little bit between each call. As it's a tad slower,
delegate it to a thread so that the stack_create RPC message doesn't
timeout when you have lots of resources.

Change-Id: I84d2b34d65b3ce7d8d858de106dac531aff509b7
Partial-Bug: #1566845
(cherry picked from commit 4a500125b350b46dee0d3c9f01c3cac7223d9c80)

8 files changed, 25 insertions, 3 deletions

diff --git a/heat/engine/stack.py b/heat/engine/stack.py
index 25616b244..599568e99 100644
--- a/heat/engine/stack.py
+++ b/heat/engine/stack.py
@@ -1261,7 +1261,7 @@ class Stack(collections.Mapping):
 
         # TODO(later): lifecycle_plugin_utils.do_pre_ops
 
-        self._converge_create_or_update()
+        self.thread_group_mgr.start(self.id, self._converge_create_or_update)
 
     def _converge_create_or_update(self):
         current_resources = self._update_or_store_resources()
@@ -1315,6 +1315,8 @@ class Stack(collections.Mapping):
                                                   self.current_traversal,
                                                   input_data, is_update,
                                                   self.adopt_stack_data)
+                if scheduler.ENABLE_SLEEP:
+                    eventlet.sleep(1)
 
     def rollback(self):
         old_tmpl_id = self.prev_raw_template_id
diff --git a/heat/tests/convergence/framework/engine_wrapper.py b/heat/tests/convergence/framework/engine_wrapper.py
index ed60ca198..0200dec53 100644
--- a/heat/tests/convergence/framework/engine_wrapper.py
+++ b/heat/tests/convergence/framework/engine_wrapper.py
@@ -114,4 +114,5 @@ class Engine(message_processor.MessageProcessor):
         cntxt = utils.dummy_context()
         db_stack = db_api.stack_get_by_name(cntxt, stack_name)
         stk = stack.Stack.load(cntxt, stack=db_stack)
+        stk.thread_group_mgr = SynchronousThreadGroupManager()
         stk.rollback()
diff --git a/heat/tests/engine/test_check_resource.py b/heat/tests/engine/test_check_resource.py
index 5d8054df5..0e8a533b3 100644
--- a/heat/tests/engine/test_check_resource.py
+++ b/heat/tests/engine/test_check_resource.py
@@ -547,6 +547,7 @@ class CheckWorkflowCleanupTest(common.HeatTestCase):
             template=tools.string_template_five, convergence=True)
         tstack.converge_stack(tstack.t, action=tstack.CREATE)
         self.stack = stack.Stack.load(self.ctx, stack_id=tstack.id)
+        self.stack.thread_group_mgr = tools.DummyThreadGroupManager()
         self.stack.converge_stack(self.stack.t, action=self.stack.DELETE)
         self.resource = self.stack['A']
         self.is_update = False
diff --git a/heat/tests/engine/tools.py b/heat/tests/engine/tools.py
index d62df271d..08c12e733 100644
--- a/heat/tests/engine/tools.py
+++ b/heat/tests/engine/tools.py
@@ -164,6 +164,7 @@ def get_stack(stack_name, ctx, template=None, with_params=True,
         tmpl = templatem.Template(t)
     stack = parser.Stack(ctx, stack_name, tmpl, convergence=convergence,
                          **kwargs)
+    stack.thread_group_mgr = DummyThreadGroupManager()
     return stack
 
 
@@ -323,6 +324,11 @@ class DummyThreadGroupManager(object):
         self.msg_queues = []
         self.messages = []
 
+    def start(self, stack, func, *args, **kwargs):
+        # Just run the function, so we know it's completed in the test
+        func(*args, **kwargs)
+        return DummyThread()
+
     def start_with_lock(self, cnxt, stack, engine_id, func, *args, **kwargs):
         # Just run the function, so we know it's completed in the test
         func(*args, **kwargs)
diff --git a/heat/tests/test_convg_stack.py b/heat/tests/test_convg_stack.py
index 7bfaf2b09..003e8950b 100644
--- a/heat/tests/test_convg_stack.py
+++ b/heat/tests/test_convg_stack.py
@@ -43,6 +43,7 @@ class StackConvergenceCreateUpdateDeleteTest(common.HeatTestCase):
         stack = parser.Stack(utils.dummy_context(), 'empty_tmpl_stack',
                              empty_tmpl, convergence=True)
         stack.store()
+        stack.thread_group_mgr = tools.DummyThreadGroupManager()
         stack.converge_stack(template=stack.t, action=stack.CREATE)
         self.assertFalse(mock_cr.called)
         mock_mc.assert_called_once_with()
@@ -175,6 +176,7 @@ class StackConvergenceCreateUpdateDeleteTest(common.HeatTestCase):
         with mock.patch.object(
                 parser.Stack, 'db_active_resources_get',
                 side_effect=self._mock_convg_db_update_requires):
+            curr_stack.thread_group_mgr = tools.DummyThreadGroupManager()
             curr_stack.converge_stack(template=template2, action=stack.UPDATE)
 
         self.assertIsNotNone(curr_stack.ext_rsrcs_db)
@@ -300,6 +302,7 @@ class StackConvergenceCreateUpdateDeleteTest(common.HeatTestCase):
         with mock.patch.object(
                 parser.Stack, 'db_active_resources_get',
                 side_effect=self._mock_convg_db_update_requires):
+            curr_stack.thread_group_mgr = tools.DummyThreadGroupManager()
             curr_stack.converge_stack(template=template2, action=stack.DELETE)
 
         self.assertIsNotNone(curr_stack.ext_rsrcs_db)
@@ -507,6 +510,7 @@ class StackConvergenceCreateUpdateDeleteTest(common.HeatTestCase):
         stack = parser.Stack(utils.dummy_context(), 'convg_updated_time_test',
                              templatem.Template.create_empty_template(),
                              convergence=True)
+        stack.thread_group_mgr = tools.DummyThreadGroupManager()
         stack.converge_stack(template=stack.t, action=stack.CREATE)
         self.assertIsNone(stack.updated_time)
         self.assertTrue(mock_ccu.called)
@@ -517,6 +521,7 @@ class StackConvergenceCreateUpdateDeleteTest(common.HeatTestCase):
                 'Resources': {'R1': {'Type': 'GenericResourceType'}}}
         stack = parser.Stack(utils.dummy_context(), 'updated_time_test',
                              templatem.Template(tmpl), convergence=True)
+        stack.thread_group_mgr = tools.DummyThreadGroupManager()
         stack.converge_stack(template=stack.t, action=stack.UPDATE)
         self.assertIsNotNone(stack.updated_time)
         self.assertTrue(mock_ccu.called)
@@ -528,6 +533,7 @@ class StackConvergenceCreateUpdateDeleteTest(common.HeatTestCase):
                                             mock_ccu, mock_cr):
         stack = parser.Stack(utils.dummy_context(), 'convg_updated_time_test',
                              templatem.Template.create_empty_template())
+        stack.thread_group_mgr = tools.DummyThreadGroupManager()
         stack.converge_stack(template=stack.t, action=stack.CREATE)
         self.assertFalse(mock_syncpoint_del.called)
         self.assertTrue(mock_ccu.called)
@@ -541,6 +547,7 @@ class StackConvergenceCreateUpdateDeleteTest(common.HeatTestCase):
                 'Resources': {'R1': {'Type': 'GenericResourceType'}}}
         stack = parser.Stack(utils.dummy_context(), 'updated_time_test',
                              templatem.Template(tmpl))
+        stack.thread_group_mgr = tools.DummyThreadGroupManager()
         stack.current_traversal = 'prev_traversal'
         stack.converge_stack(template=stack.t, action=stack.UPDATE)
         self.assertTrue(mock_syncpoint_del.called)
@@ -554,6 +561,7 @@ class StackConvergenceCreateUpdateDeleteTest(common.HeatTestCase):
         stack.current_traversal = 'prev_traversal'
         stack.action, stack.status = stack.CREATE, stack.COMPLETE
         stack.store()
+        stack.thread_group_mgr = tools.DummyThreadGroupManager()
         snapshot_values = {
             'stack_id': stack.id,
             'name': 'fake_snapshot',
diff --git a/heat/tests/test_engine_service.py b/heat/tests/test_engine_service.py
index 1b3d2ad72..d4789e7e7 100644
--- a/heat/tests/test_engine_service.py
+++ b/heat/tests/test_engine_service.py
@@ -211,6 +211,7 @@ class StackConvergenceServiceCreateUpdateTest(common.HeatTestCase):
         cfg.CONF.set_override('convergence_engine', True, enforce_type=True)
         self.ctx = utils.dummy_context()
         self.man = service.EngineService('a-host', 'a-topic')
+        self.man.thread_group_mgr = tools.DummyThreadGroupManager()
 
     def _stub_update_mocks(self, stack_to_load, stack_to_return):
         self.m.StubOutWithMock(parser, 'Stack')
diff --git a/heat/tests/test_resource.py b/heat/tests/test_resource.py
index ea63cedd0..a66343e11 100644
--- a/heat/tests/test_resource.py
+++ b/heat/tests/test_resource.py
@@ -52,6 +52,7 @@ from heat.objects import resource as resource_objects
 from heat.objects import resource_data as resource_data_object
 from heat.objects import resource_properties_data as rpd_object
 from heat.tests import common
+from heat.tests.engine import tools
 from heat.tests import generic_resource as generic_rsrc
 from heat.tests import utils
 
@@ -2074,6 +2075,7 @@ class ResourceTest(common.HeatTestCase):
             }}, env=self.env)
         stack = parser.Stack(utils.dummy_context(), 'test_stack',
                              tmpl)
+        stack.thread_group_mgr = tools.DummyThreadGroupManager()
         stack.converge_stack(stack.t, action=stack.CREATE)
         res = stack.resources['test_res']
         res.requires = [2]
@@ -2112,6 +2114,7 @@ class ResourceTest(common.HeatTestCase):
             }}, env=self.env)
         stack = parser.Stack(utils.dummy_context(), 'test_stack',
                              tmpl)
+        stack.thread_group_mgr = tools.DummyThreadGroupManager()
         stack.converge_stack(stack.t, action=stack.CREATE)
         res = stack.resources['test_res']
         res.store()
@@ -2206,6 +2209,7 @@ class ResourceTest(common.HeatTestCase):
             }}, env=self.env)
         stack = parser.Stack(utils.dummy_context(), 'test_stack',
                              tmpl)
+        stack.thread_group_mgr = tools.DummyThreadGroupManager()
         stack.converge_stack(stack.t, action=stack.CREATE)
         res = stack.resources['test_res']
         res.requires = [2]
@@ -2249,6 +2253,7 @@ class ResourceTest(common.HeatTestCase):
             }}, env=self.env)
         stack = parser.Stack(utils.dummy_context(), 'test_stack',
                              tmpl)
+        stack.thread_group_mgr = tools.DummyThreadGroupManager()
         stack.converge_stack(stack.t, action=stack.CREATE)
         res = stack.resources['test_res']
         res.requires = [2]
diff --git a/heat_integrationtests/pre_test_hook.sh b/heat_integrationtests/pre_test_hook.sh
index e6782eb33..6677b3da9 100755
--- a/heat_integrationtests/pre_test_hook.sh
+++ b/heat_integrationtests/pre_test_hook.sh
@@ -29,8 +29,6 @@ echo -e 'notification_driver=messagingv2\n' >> $localconf
 echo -e 'hidden_stack_tags=hidden\n' >> $localconf
 echo -e 'encrypt_parameters_and_properties=True\n' >> $localconf
 echo -e 'logging_exception_prefix=%(asctime)s.%(msecs)03d %(process)d TRACE %(name)s %(instance)s\n' >> $localconf
-# Limit the number of connections, we're overflowing mysql
-echo -e 'executor_thread_pool_size=8\n' >> $localconf
 
 echo -e '[heat_api]\nworkers=2\n' >> $localconf
 echo -e '[heat_api_cfn]\nworkers=2\n' >> $localconf