diff options
author | Jonathan Abrahams <jonathan@mongodb.com> | 2017-11-08 20:48:46 -0500 |
---|---|---|
committer | Jonathan Abrahams <jonathan@mongodb.com> | 2017-11-08 20:48:46 -0500 |
commit | 9dec24bdf2e1f63b0fcba6a32763e97288583e5d (patch) | |
tree | 04d078db235b8d506989e340fa3f25a6ecbc42c9 /buildscripts | |
parent | 6d8e6b9cce052cdd442e207a27df10e698b2bb00 (diff) | |
download | mongo-9dec24bdf2e1f63b0fcba6a32763e97288583e5d.tar.gz |
SERVER-31325 Powercycle new tasks:
- Replication WT
- Replication small oplog
- Sync delay
Diffstat (limited to 'buildscripts')
-rwxr-xr-x | buildscripts/aws_ec2.py | 45 |
1 files changed, 31 insertions, 14 deletions
diff --git a/buildscripts/aws_ec2.py b/buildscripts/aws_ec2.py index 2d643e045c1..6becdeaa6d6 100755 --- a/buildscripts/aws_ec2.py +++ b/buildscripts/aws_ec2.py @@ -44,24 +44,39 @@ class AwsEc2(object): @staticmethod def wait_for_state(instance, state, wait_time_secs=0, show_progress=False): """Wait up to 'wait_time_secs' for instance to be in 'state'. - Return True if 'state' reached.""" - end_time = time.time() + wait_time_secs + Return 0 if 'state' reached, 1 otherwise.""" if show_progress: print("Waiting for instance {} to reach '{}' state".format(instance, state), end="", file=sys.stdout) - while time.time() < end_time: + reached_state = False + end_time = time.time() + wait_time_secs + while True: if show_progress: print(".", end="", file=sys.stdout) sys.stdout.flush() - time.sleep(5) - instance.load() - if instance.state["Name"] == state: - if show_progress: - print(" Instance {}!".format(state), file=sys.stdout) - sys.stdout.flush() - return True - return False + try: + time_left = end_time - time.time() + instance.load() + if instance.state["Name"] == state: + reached_state = True + break + if time_left <= 0: + break + except botocore.exceptions.ClientError: + # A ClientError exception can sometimes be generated, due to RequestLimitExceeded, + # so we ignore it and retry until we time out. + pass + wait_interval_secs = 15 if time_left > 15 else time_left + time.sleep(wait_interval_secs) + if show_progress: + if reached_state: + print(" Instance {}!".format(instance.state["Name"]), file=sys.stdout) + else: + print(" Instance in state '{}', failed to reach state '{}'!".format( + instance.state["Name"], state), file=sys.stdout) + sys.stdout.flush() + return 0 if reached_state else 1 def control_instance(self, mode, image_id, wait_time_secs=0, show_progress=False): """Controls an AMI instance. Returns 0 & status information, if successful.""" @@ -69,6 +84,7 @@ class AwsEc2(object): raise ValueError( "Invalid mode '{}' specified, choose from {}.".format(mode, _MODES)) + sys.stdout.flush() instance = self.connection.Instance(image_id) try: if mode == "start": @@ -92,8 +108,9 @@ class AwsEc2(object): except botocore.exceptions.ClientError as err: return 1, err.message + ret = 0 if wait_time_secs > 0: - self.wait_for_state( + ret = self.wait_for_state( instance=instance, state=state, wait_time_secs=wait_time_secs, @@ -113,7 +130,7 @@ class AwsEc2(object): except botocore.exceptions.ClientError as err: return 1, err.message - return 0, status + return ret, status def tag_instance(self, image_id, tags): """Tags an AMI instance. """ @@ -208,7 +225,7 @@ def main(): control_options.add_option("--waitTimeSecs", dest="wait_time_secs", type=int, - default=60, + default=5 * 60, help="Time to wait for EC2 instance to reach it's new state," " defaults to '%default'.") |