diff options
| author | Zuul <zuul@review.opendev.org> | 2019-11-15 06:19:54 +0000 |
|---|---|---|
| committer | Gerrit Code Review <review@openstack.org> | 2019-11-15 06:19:54 +0000 |
| commit | 28433d63d8ecc5fd57519d879089318f229e9317 (patch) | |
| tree | af140f5644a9297e5af50750e949e0226a309df9 | |
| parent | f2f2f3f0a91791384aa35719725b44de27d133fc (diff) | |
| parent | 33cdf77cc7f26da36447e8d223eba29d9678a02c (diff) | |
| download | heat-28433d63d8ecc5fd57519d879089318f229e9317.tar.gz | |
Merge "Add retries when loading keystone data and fetching endpoints" into stable/queens
| -rw-r--r-- | heat/common/context.py | 13 | ||||
| -rw-r--r-- | heat/engine/clients/client_plugin.py | 2 |
2 files changed, 15 insertions, 0 deletions
diff --git a/heat/common/context.py b/heat/common/context.py index ad89c83f6..dee693c47 100644 --- a/heat/common/context.py +++ b/heat/common/context.py @@ -12,6 +12,7 @@ # under the License. from keystoneauth1 import access +from keystoneauth1 import exceptions as ksa_exceptions from keystoneauth1.identity import access as access_plugin from keystoneauth1.identity import generic from keystoneauth1 import loading as ks_loading @@ -24,6 +25,7 @@ import oslo_messaging from oslo_middleware import request_id as oslo_request_id from oslo_utils import importutils import six +import tenacity from heat.common import config from heat.common import endpoint_utils @@ -51,6 +53,15 @@ TRUSTEE_CONF_GROUP = 'trustee' ks_loading.register_auth_conf_options(cfg.CONF, TRUSTEE_CONF_GROUP) +retry_on_connection_timeout = tenacity.retry( + stop=tenacity.stop_after_attempt(cfg.CONF.client_retry_limit+1), + wait=tenacity.wait_random(max=2), + retry=tenacity.retry_if_exception_type( + (ksa_exceptions.ConnectFailure, + ksa_exceptions.DiscoveryFailure)), + reraise=True) + + def list_opts(): trustee_opts = ks_loading.get_auth_common_conf_options() trustee_opts.extend(ks_loading.get_auth_plugin_conf_options( @@ -287,6 +298,8 @@ class RequestContext(context.RequestContext): class StoredContext(RequestContext): + + @retry_on_connection_timeout def _load_keystone_data(self): self._keystone_loaded = True auth_ref = self.auth_plugin.get_access(self.keystone_session) diff --git a/heat/engine/clients/client_plugin.py b/heat/engine/clients/client_plugin.py index ab737da6f..6f485d56a 100644 --- a/heat/engine/clients/client_plugin.py +++ b/heat/engine/clients/client_plugin.py @@ -24,6 +24,7 @@ import requests import six from heat.common import config +from heat.common import context from heat.common import exception as heat_exception cfg.CONF.import_opt('client_retry_limit', 'heat.common.config') @@ -93,6 +94,7 @@ class ClientPlugin(object): def url_for(self, **kwargs): keystone_session = self.context.keystone_session + @context.retry_on_connection_timeout def get_endpoint(): return keystone_session.get_endpoint(**kwargs) |