diff options
| author | Mick Bass <mick.bass@47lining.com> | 2014-10-25 00:16:10 -0700 |
|---|---|---|
| committer | Mick Bass <mick.bass@47lining.com> | 2014-12-25 14:37:08 -0700 |
| commit | 2d189f0d192717f83e3c6d37d3fe0988fc329b5a (patch) | |
| tree | 15c7e93a8c3edd1212dfda0e96587127523d497e | |
| parent | 170457413dd179c3154a4184cbe12ad1ab14c86e (diff) | |
| download | ansible-modules-core-2d189f0d192717f83e3c6d37d3fe0988fc329b5a.tar.gz | |
add retry with exponential backoff when we receive throttling error code from cloudformation
| -rw-r--r-- | cloud/amazon/cloudformation.py | 19 |
1 files changed, 16 insertions, 3 deletions
diff --git a/cloud/amazon/cloudformation.py b/cloud/amazon/cloudformation.py index b382e3f0..6963fb6f 100644 --- a/cloud/amazon/cloudformation.py +++ b/cloud/amazon/cloudformation.py @@ -160,7 +160,7 @@ def stack_operation(cfn, stack_name, operation): operation_complete = False while operation_complete == False: try: - stack = cfn.describe_stacks(stack_name)[0] + stack = invoke_with_throttling_retries(cfn.describe_stacks, stack_name)[0] existed.append('yes') except: if 'yes' in existed: @@ -189,6 +189,19 @@ def stack_operation(cfn, stack_name, operation): time.sleep(5) return result +IGNORE_CODE = 'Throttling' +MAX_RETRIES=3 +def invoke_with_throttling_retries(function_ref, *argv): + retries=0 + while True: + try: + retval=function_ref(*argv) + return retval + except boto.exception.BotoServerError, e: + if e.code != IGNORE_CODE or retries==MAX_RETRIES: + raise e + time.sleep(5 * (2**retries)) + retries += 1 def main(): argument_spec = ec2_argument_spec() @@ -288,7 +301,7 @@ def main(): # and get the outputs of the stack if state == 'present' or update: - stack = cfn.describe_stacks(stack_name)[0] + stack = invoke_with_throttling_retries(cfn.describe_stacks,stack_name)[0] for output in stack.outputs: stack_outputs[output.key] = output.value result['stack_outputs'] = stack_outputs @@ -299,7 +312,7 @@ def main(): if state == 'absent': try: - cfn.describe_stacks(stack_name) + invoke_with_throttling_retries(cfn.describe_stacks,stack_name) operation = 'DELETE' except Exception, err: error_msg = boto_exception(err) |