From 775acf52589647014ca916f2f6e43587a20f5f0b Mon Sep 17 00:00:00 2001 From: Steve Baker Date: Fri, 18 Dec 2015 09:10:46 +1300 Subject: Make minimum default num_engine_workers>=4 Downstream test environments are frequently having failing stacks with error messages like: MessagingTimeout: resources[0]: Timed out waiting for a reply to message ID ... These environments generally have 1 or 2 cores, so only spawn one or two engine workers. This deadlocks with stacks that have many nested stacks due to engine->engine RPC calls. Even our own functional tests don't work reliably with less than 4 workers, and the workaround has been to set that explicitly in pre_test_hook.sh. This change sets the default minimum number of workers to 4, but still matches workers to cores for larger servers. This change also moves the default evaluation to heat.cmd.engine so that generated configuration doesn't get a inappropriate default value. Change-Id: Iae6b3956bad414406d901bb2213c9ec230ff4304 Closes-Bug: #1526045 (cherry picked from commit adb21217955e59fce5fb194635b36b5b40d6d8c8) --- heat/cmd/engine.py | 8 ++++++-- heat/common/config.py | 2 -- 2 files changed, 6 insertions(+), 4 deletions(-) (limited to 'heat') diff --git a/heat/cmd/engine.py b/heat/cmd/engine.py index 1574a5a60..bee78f589 100755 --- a/heat/cmd/engine.py +++ b/heat/cmd/engine.py @@ -23,6 +23,7 @@ eventlet.monkey_patch() import sys +from oslo_concurrency import processutils from oslo_config import cfg import oslo_i18n as i18n from oslo_log import log as logging @@ -65,8 +66,11 @@ def main(): profiler.setup('heat-engine', cfg.CONF.host) gmr.TextGuruMeditation.setup_autorun(version) srv = engine.EngineService(cfg.CONF.host, rpc_api.ENGINE_TOPIC) - launcher = service.launch(cfg.CONF, srv, - workers=cfg.CONF.num_engine_workers) + workers = cfg.CONF.num_engine_workers + if not workers: + workers = max(4, processutils.get_worker_count()) + + launcher = service.launch(cfg.CONF, srv, workers=workers) if cfg.CONF.enable_cloud_watch_lite: # We create the periodic tasks here, which mean they are created # only in the parent process when num_engine_workers>1 is specified diff --git a/heat/common/config.py b/heat/common/config.py index b59815e9a..86c9f0e53 100644 --- a/heat/common/config.py +++ b/heat/common/config.py @@ -18,7 +18,6 @@ import logging as sys_logging import os from eventlet.green import socket -from oslo_concurrency import processutils from oslo_config import cfg from oslo_log import log as logging @@ -84,7 +83,6 @@ service_opts = [ default=5, help=_('Maximum depth allowed when using nested stacks.')), cfg.IntOpt('num_engine_workers', - default=processutils.get_worker_count(), help=_('Number of heat-engine processes to fork and run.'))] engine_opts = [ -- cgit v1.2.1