Make minimum default num_engine_workers>=4

Downstream test environments are frequently having failing stacks with
error messages like:

  MessagingTimeout: resources[0]: Timed out waiting for a reply to
  message ID ...

These environments generally have 1 or 2 cores, so only spawn one or two
engine workers. This deadlocks with stacks that have many nested stacks
due to engine->engine RPC calls.

Even our own functional tests don't work reliably with less than 4
workers, and the workaround has been to set that explicitly in
pre_test_hook.sh.

This change sets the default minimum number of workers to 4, but still
matches workers to cores for larger servers.

This change also moves the default evaluation to heat.cmd.engine so that
generated configuration doesn't get a inappropriate default value.

Change-Id: Iae6b3956bad414406d901bb2213c9ec230ff4304
Closes-Bug: #1526045
(cherry picked from commit adb21217955e59fce5fb194635b36b5b40d6d8c8)

3 files changed, 6 insertions, 4 deletions

diff --git a/bin/heat-engine b/bin/heat-engine
index 5f43784c0..dccde9526 100755
--- a/bin/heat-engine
+++ b/bin/heat-engine
@@ -32,6 +32,7 @@ POSSIBLE_TOPDIR = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
 if os.path.exists(os.path.join(POSSIBLE_TOPDIR, 'heat', '__init__.py')):
     sys.path.insert(0, POSSIBLE_TOPDIR)
 
+from oslo_concurrency import processutils
 from oslo_config import cfg
 import oslo_i18n as i18n
 from oslo_log import log as logging
@@ -71,7 +72,11 @@ if __name__ == '__main__':
 
     profiler.setup('heat-engine', cfg.CONF.host)
     srv = engine.EngineService(cfg.CONF.host, rpc_api.ENGINE_TOPIC)
-    launcher = service.launch(srv, workers=cfg.CONF.num_engine_workers)
+    workers = cfg.CONF.num_engine_workers
+    if not workers:
+        workers = max(4, processutils.get_worker_count())
+
+    launcher = service.launch(srv, workers=workers)
     if cfg.CONF.enable_cloud_watch_lite:
         # We create the periodic tasks here, which mean they are created
         # only in the parent process when num_engine_workers>1 is specified
diff --git a/heat/common/config.py b/heat/common/config.py
index 806eae760..ebf8ad006 100644
--- a/heat/common/config.py
+++ b/heat/common/config.py
@@ -18,7 +18,6 @@ import logging as sys_logging
 import os
 
 from eventlet.green import socket
-from oslo_concurrency import processutils
 from oslo_config import cfg
 from oslo_log import log as logging
 
@@ -85,7 +84,6 @@ service_opts = [
                default=5,
                help=_('Maximum depth allowed when using nested stacks.')),
     cfg.IntOpt('num_engine_workers',
-               default=processutils.get_worker_count(),
                help=_('Number of heat-engine processes to fork and run.'))]
 
 engine_opts = [
diff --git a/heat_integrationtests/pre_test_hook.sh b/heat_integrationtests/pre_test_hook.sh
index fa4db09d2..17718363c 100755
--- a/heat_integrationtests/pre_test_hook.sh
+++ b/heat_integrationtests/pre_test_hook.sh
@@ -19,5 +19,4 @@ localconf=$BASE/new/devstack/local.conf
 echo "HEAT_ENABLE_ADOPT_ABANDON=True" >> $localrc_path
 echo -e '[[post-config|$HEAT_CONF]]\n[DEFAULT]\n' >> $localconf
 echo -e 'notification_driver=messagingv2\n' >> $localconf
-echo -e 'num_engine_workers=2\n' >> $localconf
 echo -e 'plugin_dirs=$HEAT_DIR/heat_integrationtests/common/test_resources\n' >> $localconf