diff options
author | Brian Coca <bcoca@ansible.com> | 2015-03-27 08:21:09 -0400 |
---|---|---|
committer | Brian Coca <bcoca@ansible.com> | 2015-03-27 08:21:09 -0400 |
commit | f3e4704d92b051542b0523f753384bef9b849639 (patch) | |
tree | 381c9c5d452c405bd84e4cbfa10ec3caf04a91aa | |
parent | aeb264f5816e7f996ce94ebab6385c7d8c05bbc4 (diff) | |
parent | 2d189f0d192717f83e3c6d37d3fe0988fc329b5a (diff) | |
download | ansible-modules-core-f3e4704d92b051542b0523f753384bef9b849639.tar.gz |
Merge pull request #569 from 47lining/rate-throttling-retries-update
add retry with exponential backoff when we receive throttling error code...
-rw-r--r-- | cloud/amazon/cloudformation.py | 19 |
1 files changed, 16 insertions, 3 deletions
diff --git a/cloud/amazon/cloudformation.py b/cloud/amazon/cloudformation.py index e1a16b99..905b96ea 100644 --- a/cloud/amazon/cloudformation.py +++ b/cloud/amazon/cloudformation.py @@ -142,7 +142,7 @@ def stack_operation(cfn, stack_name, operation): operation_complete = False while operation_complete == False: try: - stack = cfn.describe_stacks(stack_name)[0] + stack = invoke_with_throttling_retries(cfn.describe_stacks, stack_name)[0] existed.append('yes') except: if 'yes' in existed: @@ -171,6 +171,19 @@ def stack_operation(cfn, stack_name, operation): time.sleep(5) return result +IGNORE_CODE = 'Throttling' +MAX_RETRIES=3 +def invoke_with_throttling_retries(function_ref, *argv): + retries=0 + while True: + try: + retval=function_ref(*argv) + return retval + except boto.exception.BotoServerError, e: + if e.code != IGNORE_CODE or retries==MAX_RETRIES: + raise e + time.sleep(5 * (2**retries)) + retries += 1 def main(): argument_spec = ec2_argument_spec() @@ -271,7 +284,7 @@ def main(): # and get the outputs of the stack if state == 'present' or update: - stack = cfn.describe_stacks(stack_name)[0] + stack = invoke_with_throttling_retries(cfn.describe_stacks,stack_name)[0] for output in stack.outputs: stack_outputs[output.key] = output.value result['stack_outputs'] = stack_outputs @@ -282,7 +295,7 @@ def main(): if state == 'absent': try: - cfn.describe_stacks(stack_name) + invoke_with_throttling_retries(cfn.describe_stacks,stack_name) operation = 'DELETE' except Exception, err: error_msg = boto_exception(err) |