[cloud] Retry WAF actions on WAFStaleDataException (#36405) (#36507)

Add a util to run functions with AWSRetry to retry on WAFStaleDataExceptions and update ChangeToken for each attempt

5 files changed, 65 insertions, 39 deletions

diff --git a/lib/ansible/module_utils/aws/waf.py b/lib/ansible/module_utils/aws/waf.py
index dfdb327b42..6758b40ed6 100644
--- a/lib/ansible/module_utils/aws/waf.py
+++ b/lib/ansible/module_utils/aws/waf.py
@@ -180,3 +180,9 @@ def get_change_token(client, module):
         return token['ChangeToken']
     except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
         module.fail_json_aws(e, msg="Couldn't obtain change token")
+
+
+@AWSRetry.backoff(tries=10, delay=2, backoff=2.0, catch_extra_error_codes=['WAFStaleDataException'])
+def run_func_with_change_token_backoff(client, module, params, func):
+    params['ChangeToken'] = get_change_token(client, module)
+    return func(**params)
diff --git a/lib/ansible/modules/cloud/amazon/aws_waf_condition.py b/lib/ansible/modules/cloud/amazon/aws_waf_condition.py
index 4565a0d709..4ae88d1d1d 100644
--- a/lib/ansible/modules/cloud/amazon/aws_waf_condition.py
+++ b/lib/ansible/modules/cloud/amazon/aws_waf_condition.py
@@ -331,7 +331,7 @@ except ImportError:
 from ansible.module_utils.aws.core import AnsibleAWSModule
 from ansible.module_utils.ec2 import boto3_conn, get_aws_connection_info, ec2_argument_spec
 from ansible.module_utils.ec2 import camel_dict_to_snake_dict, AWSRetry, compare_policies
-from ansible.module_utils.aws.waf import get_change_token, MATCH_LOOKUP
+from ansible.module_utils.aws.waf import run_func_with_change_token_backoff, MATCH_LOOKUP
 from ansible.module_utils.aws.waf import get_rule_with_backoff, list_rules_with_backoff
 
 
@@ -397,12 +397,10 @@ class Condition(object):
             kwargs['Updates'].append({'Action': 'INSERT', self.conditiontuple: condition_insert})
 
         kwargs[self.conditionsetid] = condition_set_id
-        kwargs['ChangeToken'] = get_change_token(self.client, self.module)
         return kwargs
 
     def format_for_deletion(self, condition):
-        return {'ChangeToken': get_change_token(self.client, self.module),
-                'Updates': [{'Action': 'DELETE', self.conditiontuple: current_condition_tuple}
+        return {'Updates': [{'Action': 'DELETE', self.conditiontuple: current_condition_tuple}
                             for current_condition_tuple in condition[self.conditiontuples]],
                 self.conditionsetid: condition[self.conditionsetid]}
 
@@ -443,15 +441,17 @@ class Condition(object):
 
         pattern_set = self.get_regex_pattern_by_name(name)
         if not pattern_set:
-            pattern_set = self.client.create_regex_pattern_set(Name=name, ChangeToken=get_change_token(self.client, self.module))['RegexPatternSet']
+            pattern_set = run_func_with_change_token_backoff(self.client, self.module, {'Name': name},
+                                                             self.client.create_regex_pattern_set)['RegexPatternSet']
         missing = set(regex_pattern['regex_strings']) - set(pattern_set['RegexPatternStrings'])
         extra = set(pattern_set['RegexPatternStrings']) - set(regex_pattern['regex_strings'])
         if not missing and not extra:
             return pattern_set
         updates = [{'Action': 'INSERT', 'RegexPatternString': pattern} for pattern in missing]
         updates.extend([{'Action': 'DELETE', 'RegexPatternString': pattern} for pattern in extra])
-        self.client.update_regex_pattern_set(RegexPatternSetId=pattern_set['RegexPatternSetId'],
-                                             Updates=updates, ChangeToken=get_change_token(self.client, self.module))
+        run_func_with_change_token_backoff(self.client, self.module,
+                                           {'RegexPatternSetId': pattern_set['RegexPatternSetId'], 'Updates': updates},
+                                           self.client.update_regex_pattern_set)
         return self.get_regex_pattern_set_with_backoff(pattern_set['RegexPatternSetId'])['RegexPatternSet']
 
     def delete_unused_regex_pattern(self, regex_pattern_set_id):
@@ -460,11 +460,13 @@ class Condition(object):
             updates = list()
             for regex_pattern_string in regex_pattern_set['RegexPatternStrings']:
                 updates.append({'Action': 'DELETE', 'RegexPatternString': regex_pattern_string})
-            self.client.update_regex_pattern_set(RegexPatternSetId=regex_pattern_set_id, Updates=updates,
-                                                 ChangeToken=get_change_token(self.client, self.module))
+            run_func_with_change_token_backoff(self.client, self.module,
+                                               {'RegexPatternSetId': regex_pattern_set_id, 'Updates': updates},
+                                               self.client.update_regex_pattern_set)
 
-            self.client.delete_regex_pattern_set(RegexPatternSetId=regex_pattern_set_id,
-                                                 ChangeToken=get_change_token(self.client, self.module))
+            run_func_with_change_token_backoff(self.client, self.module,
+                                               {'RegexPatternSetId': regex_pattern_set_id},
+                                               self.client.delete_regex_pattern_set)
         except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
             self.module.fail_json_aws(e, msg='Could not delete regex pattern')
 
@@ -535,15 +537,14 @@ class Condition(object):
             func = getattr(self.client, 'update_' + self.method_suffix)
             params = self.format_for_deletion(current_condition)
             try:
-                func(**params)
+                run_func_with_change_token_backoff(self.client, self.module, params, func)
             except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
                 self.module.fail_json_aws(e, msg='Could not delete filters from condition')
         func = getattr(self.client, 'delete_' + self.method_suffix)
         params = dict()
         params[self.conditionsetid] = condition_set_id
-        params['ChangeToken'] = get_change_token(self.client, self.module)
         try:
-            func(**params)
+            run_func_with_change_token_backoff(self.client, self.module, params, func)
         except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
             self.module.fail_json_aws(e, msg='Could not delete condition')
         # tidy up regex patterns
@@ -579,7 +580,7 @@ class Condition(object):
             update['Updates'] = missing + extra
             func = getattr(self.client, 'update_' + self.method_suffix)
             try:
-                func(**update)
+                run_func_with_change_token_backoff(self.client, self.module, update, func)
             except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
                 self.module.fail_json_aws(e, msg='Could not update condition')
         return changed, self.get_condition_by_id(condition_set_id)
@@ -592,10 +593,9 @@ class Condition(object):
         else:
             params = dict()
             params['Name'] = name
-            params['ChangeToken'] = get_change_token(self.client, self.module)
             func = getattr(self.client, 'create_' + self.method_suffix)
             try:
-                condition = func(**params)
+                condition = run_func_with_change_token_backoff(self.client, self.module, params, func)
             except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
                 self.module.fail_json_aws(e, msg='Could not create condition')
             return self.find_and_update_condition(condition[self.conditionset][self.conditionsetid])
diff --git a/lib/ansible/modules/cloud/amazon/aws_waf_rule.py b/lib/ansible/modules/cloud/amazon/aws_waf_rule.py
index 1593cb3117..afc5fab15e 100644
--- a/lib/ansible/modules/cloud/amazon/aws_waf_rule.py
+++ b/lib/ansible/modules/cloud/amazon/aws_waf_rule.py
@@ -126,7 +126,7 @@ except ImportError:
 from ansible.module_utils.aws.core import AnsibleAWSModule
 from ansible.module_utils.ec2 import boto3_conn, get_aws_connection_info, ec2_argument_spec
 from ansible.module_utils.ec2 import camel_dict_to_snake_dict
-from ansible.module_utils.aws.waf import get_change_token, list_rules_with_backoff, MATCH_LOOKUP
+from ansible.module_utils.aws.waf import run_func_with_change_token_backoff, list_rules_with_backoff, MATCH_LOOKUP
 from ansible.module_utils.aws.waf import get_web_acl_with_backoff, list_web_acls_with_backoff
 
 
@@ -201,10 +201,13 @@ def find_and_update_rule(client, module, rule_id):
                               if not all_conditions[condition_type][condition['data_id']]['name'] in desired_conditions[condition_type]])
 
     changed = bool(insertions or deletions)
+    update = {
+        'RuleId': rule_id,
+        'Updates': insertions + deletions
+    }
     if changed:
         try:
-            client.update_rule(RuleId=rule_id, ChangeToken=get_change_token(client, module),
-                               Updates=insertions + deletions)
+            run_func_with_change_token_backoff(client, module, update, client.update_rule)
         except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
             module.fail_json_aws(e, msg='Could not update rule conditions')
 
@@ -229,8 +232,7 @@ def remove_rule_conditions(client, module, rule_id):
     conditions = get_rule(client, module, rule_id)['Predicates']
     updates = [format_for_deletion(camel_dict_to_snake_dict(condition)) for condition in conditions]
     try:
-        client.update_rule(RuleId=rule_id,
-                           ChangeToken=get_change_token(client, module), Updates=updates)
+        run_func_with_change_token_backoff(client, module, {'RuleId': rule_id, 'Updates': updates}, client.update_rule)
     except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
         module.fail_json_aws(e, msg='Could not remove rule conditions')
 
@@ -247,9 +249,8 @@ def ensure_rule_present(client, module):
         if not metric_name:
             metric_name = re.sub(r'[^a-zA-Z0-9]', '', module.params['name'])
         params['MetricName'] = metric_name
-        params['ChangeToken'] = get_change_token(client, module)
         try:
-            new_rule = client.create_rule(**params)['Rule']
+            new_rule = run_func_with_change_token_backoff(client, module, params, client.create_rule)['Rule']
         except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
             module.fail_json_aws(e, msg='Could not create rule')
         return find_and_update_rule(client, module, new_rule['RuleId'])
@@ -281,7 +282,7 @@ def ensure_rule_absent(client, module):
     if rule_id:
         remove_rule_conditions(client, module, rule_id)
         try:
-            return True, client.delete_rule(RuleId=rule_id, ChangeToken=get_change_token(client, module))
+            return True, run_func_with_change_token_backoff(client, module, {'RuleId': rule_id}, client.delete_rule)
         except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
             module.fail_json_aws(e, msg='Could not delete rule')
     return False, {}
diff --git a/lib/ansible/modules/cloud/amazon/aws_waf_web_acl.py b/lib/ansible/modules/cloud/amazon/aws_waf_web_acl.py
index 0e705d816f..2b0ea67714 100644
--- a/lib/ansible/modules/cloud/amazon/aws_waf_web_acl.py
+++ b/lib/ansible/modules/cloud/amazon/aws_waf_web_acl.py
@@ -136,7 +136,7 @@ import re
 
 from ansible.module_utils.aws.core import AnsibleAWSModule
 from ansible.module_utils.ec2 import boto3_conn, get_aws_connection_info, ec2_argument_spec, camel_dict_to_snake_dict
-from ansible.module_utils.aws.waf import list_rules_with_backoff, list_web_acls_with_backoff, get_change_token
+from ansible.module_utils.aws.waf import list_rules_with_backoff, list_web_acls_with_backoff, run_func_with_change_token_backoff
 
 
 def get_web_acl_by_name(client, module, name):
@@ -186,16 +186,26 @@ def find_and_update_web_acl(client, module, web_acl_id):
     insertions = [format_for_update(rule, 'INSERT') for rule in missing]
     deletions = [format_for_update(rule, 'DELETE') for rule in extras]
     changed = bool(insertions + deletions)
-    if changed:
+
+    # Purge rules before adding new ones in case a deletion shares the same
+    # priority as an insertion.
+    params = {
+        'WebACLId': acl['WebACLId'],
+        'DefaultAction': acl['DefaultAction']
+    }
+    if deletions:
         try:
-            client.update_web_acl(
-                WebACLId=acl['WebACLId'],
-                ChangeToken=get_change_token(client, module),
-                Updates=insertions + deletions,
-                DefaultAction=acl['DefaultAction']
-            )
+            params['Updates'] = deletions
+            run_func_with_change_token_backoff(client, module, params, client.update_web_acl)
         except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
             module.fail_json_aws(e, msg='Could not update Web ACL')
+    if insertions:
+        try:
+            params['Updates'] = insertions
+            run_func_with_change_token_backoff(client, module, params, client.update_web_acl)
+        except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+            module.fail_json_aws(e, msg='Could not update Web ACL')
+    if changed:
         acl = get_web_acl(client, module, web_acl_id)
     return changed, acl
 
@@ -217,8 +227,8 @@ def remove_rules_from_web_acl(client, module, web_acl_id):
     acl = get_web_acl(client, module, web_acl_id)
     deletions = [format_for_update(rule, 'DELETE') for rule in acl['Rules']]
     try:
-        client.update_web_acl(WebACLId=acl['WebACLId'], ChangeToken=get_change_token(client, module),
-                              Updates=deletions, DefaultAction=acl['DefaultAction'])
+        params = {'WebACLId': acl['WebACLId'], 'DefaultAction': acl['DefaultAction'], 'Updates': deletions}
+        run_func_with_change_token_backoff(client, module, params, client.update_web_acl)
     except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
         module.fail_json_aws(e, msg='Could not remove rule')
 
@@ -236,9 +246,8 @@ def ensure_web_acl_present(client, module):
             metric_name = re.sub(r'[^A-Za-z0-9]', '', module.params['name'])
         default_action = module.params['default_action'].upper()
         try:
-            new_web_acl = client.create_web_acl(Name=name, MetricName=metric_name,
-                                                DefaultAction={'Type': default_action},
-                                                ChangeToken=get_change_token(client, module))
+            params = {'Name': name, 'MetricName': metric_name, 'DefaultAction': {'Type': default_action}}
+            new_web_acl = run_func_with_change_token_backoff(client, module, params, client.create_web_acl)
         except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
             module.fail_json_aws(e, msg='Could not create Web ACL')
         (changed, result) = find_and_update_web_acl(client, module, new_web_acl['WebACL']['WebACLId'])
@@ -252,7 +261,7 @@ def ensure_web_acl_absent(client, module):
         if web_acl['Rules']:
             remove_rules_from_web_acl(client, module, web_acl_id)
         try:
-            client.delete_web_acl(WebACLId=web_acl_id, ChangeToken=get_change_token(client, module))
+            run_func_with_change_token_backoff(client, module, {'WebACLId': web_acl_id}, client.delete_web_acl)
             return True, {}
         except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
             module.fail_json_aws(e, msg='Could not delete Web ACL')
diff --git a/test/integration/targets/aws_waf_web_acl/tasks/main.yml b/test/integration/targets/aws_waf_web_acl/tasks/main.yml
index faf5cc88f1..e1dac5fca7 100644
--- a/test/integration/targets/aws_waf_web_acl/tasks/main.yml
+++ b/test/integration/targets/aws_waf_web_acl/tasks/main.yml
@@ -481,10 +481,19 @@
   - debug:
       msg: "****** TEARDOWN STARTS HERE ******"
 
+  - name: delete the web acl
+    aws_waf_web_acl:
+      name: "{{ resource_prefix }}_web_acl"
+      state: absent
+      purge_rules: yes
+      <<: *aws_connection_info
+    ignore_errors: yes
+
   - name: remove second WAF rule
     aws_waf_rule:
       name: "{{ resource_prefix }}_rule_2"
       state: absent
+      purge_conditions: yes
       <<: *aws_connection_info
     ignore_errors: yes
 
@@ -492,6 +501,7 @@
     aws_waf_rule:
       name: "{{ resource_prefix }}_rule"
       state: absent
+      purge_conditions: yes
       <<: *aws_connection_info
     ignore_errors: yes