diff options
| author | Rabi Mishra <ramishra@redhat.com> | 2019-08-23 10:45:28 +0530 |
|---|---|---|
| committer | Rabi Mishra <ramishra@redhat.com> | 2019-10-23 10:19:57 +0000 |
| commit | 33cdf77cc7f26da36447e8d223eba29d9678a02c (patch) | |
| tree | 6199254e91f1e165f35aee007072fc92ad2d8516 | |
| parent | 6d0392c85c42bafbd4b0618390a68f0cb8174f04 (diff) | |
| download | heat-33cdf77cc7f26da36447e8d223eba29d9678a02c.tar.gz | |
Add retries when loading keystone data and fetching endpoints
We may end up loading lots of nested stacks concurrently with
convergence and those would try to discover endpoints and fetch
access info objects from keystone. This at times results in
ConnectTimeout errors from keystone. We can avoid these errors
by adding some retries.
Also adds retries to client_plugin get_endpoint() calls, which has
simillar issue.
Change-Id: I18cde971248eff5783f97c9e7a60316d7dd93431
Task: 36349
(cherry picked from commit 6fb8ac250ae27810516b89f273923d6b786606b5)
| -rw-r--r-- | heat/common/context.py | 13 | ||||
| -rw-r--r-- | heat/engine/clients/client_plugin.py | 2 |
2 files changed, 15 insertions, 0 deletions
diff --git a/heat/common/context.py b/heat/common/context.py index ad89c83f6..dee693c47 100644 --- a/heat/common/context.py +++ b/heat/common/context.py @@ -12,6 +12,7 @@ # under the License. from keystoneauth1 import access +from keystoneauth1 import exceptions as ksa_exceptions from keystoneauth1.identity import access as access_plugin from keystoneauth1.identity import generic from keystoneauth1 import loading as ks_loading @@ -24,6 +25,7 @@ import oslo_messaging from oslo_middleware import request_id as oslo_request_id from oslo_utils import importutils import six +import tenacity from heat.common import config from heat.common import endpoint_utils @@ -51,6 +53,15 @@ TRUSTEE_CONF_GROUP = 'trustee' ks_loading.register_auth_conf_options(cfg.CONF, TRUSTEE_CONF_GROUP) +retry_on_connection_timeout = tenacity.retry( + stop=tenacity.stop_after_attempt(cfg.CONF.client_retry_limit+1), + wait=tenacity.wait_random(max=2), + retry=tenacity.retry_if_exception_type( + (ksa_exceptions.ConnectFailure, + ksa_exceptions.DiscoveryFailure)), + reraise=True) + + def list_opts(): trustee_opts = ks_loading.get_auth_common_conf_options() trustee_opts.extend(ks_loading.get_auth_plugin_conf_options( @@ -287,6 +298,8 @@ class RequestContext(context.RequestContext): class StoredContext(RequestContext): + + @retry_on_connection_timeout def _load_keystone_data(self): self._keystone_loaded = True auth_ref = self.auth_plugin.get_access(self.keystone_session) diff --git a/heat/engine/clients/client_plugin.py b/heat/engine/clients/client_plugin.py index ab737da6f..6f485d56a 100644 --- a/heat/engine/clients/client_plugin.py +++ b/heat/engine/clients/client_plugin.py @@ -24,6 +24,7 @@ import requests import six from heat.common import config +from heat.common import context from heat.common import exception as heat_exception cfg.CONF.import_opt('client_retry_limit', 'heat.common.config') @@ -93,6 +94,7 @@ class ClientPlugin(object): def url_for(self, **kwargs): keystone_session = self.context.keystone_session + @context.retry_on_connection_timeout def get_endpoint(): return keystone_session.get_endpoint(**kwargs) |