Avoid double-check when waiting for test states

Sometimes tests fail becaused there's a double check for
the instance state.  If the state changes in between the
two checks, the test will fail to acknowledge the first
'correct' response and timeout while waiting for the state
to re-occur (which doesn't happen).

Sample output would look like:
[REPORT] Polling instance '<id>' for state 'REBOOT', was 'REBOOT'.
[REPORT] Polling instance '<id>' for state 'REBOOT', was 'ACTIVE'.
...
[REPORT] Polling instance '<id>' for state 'REBOOT', was 'ACTIVE'.
...
[REPORT] Status of instance '<id>' did not change to 'REBOOT' after 1201.3s.

Also broke down the instance-create tests into instance-create and
instance-init-create so that the initialzed instance won't
need to be created for manual runs or multi runs in the gate.

Reorganized the backup create so that on fast nodes there's
more chance that the backup is still around while the
negative tests run.

Change-Id: I9731bf7a93e7b8b3398cf73d1bc15d70f5bfa96a

9 files changed, 86 insertions, 52 deletions

diff --git a/trove/tests/api/backups.py b/trove/tests/api/backups.py
index 9046a0bb..a4ebd6f7 100644
--- a/trove/tests/api/backups.py
+++ b/trove/tests/api/backups.py
@@ -80,7 +80,8 @@ class CreateBackups(object):
         assert_raises(exceptions.NotFound, instance_info.dbaas.backups.create,
                       BACKUP_NAME, generate_uuid(), BACKUP_DESC)
 
-    @test
+    @test(runs_after=[test_backup_create_instance_invalid,
+                      test_backup_create_instance_not_found])
     def test_backup_create_instance(self):
         """Test create backup for a given instance."""
         # Necessary to test that the count increases.
@@ -89,6 +90,9 @@ class CreateBackups(object):
         global backup_count_for_instance_prior_to_create
         backup_count_for_instance_prior_to_create = len(
             instance_info.dbaas.instances.backups(instance_info.id))
+        datastore_version = instance_info.dbaas.datastore_versions.get(
+            instance_info.dbaas_datastore,
+            instance_info.dbaas_datastore_version)
 
         result = instance_info.dbaas.backups.create(BACKUP_NAME,
                                                     instance_info.id,
@@ -101,10 +105,6 @@ class CreateBackups(object):
         assert_equal('NEW', result.status)
         instance = instance_info.dbaas.instances.get(instance_info.id)
 
-        datastore_version = instance_info.dbaas.datastore_versions.get(
-            instance_info.dbaas_datastore,
-            instance_info.dbaas_datastore_version)
-
         assert_equal('BACKUP', instance.status)
         assert_equal(instance_info.dbaas_datastore,
                      result.datastore['type'])
diff --git a/trove/tests/int_tests.py b/trove/tests/int_tests.py
index 0a092876..c87466fc 100644
--- a/trove/tests/int_tests.py
+++ b/trove/tests/int_tests.py
@@ -154,18 +154,22 @@ cluster_actions_groups.extend([cluster_actions_group.GROUP,
 
 # Single-instance based groups
 instance_create_groups = list(base_groups)
-instance_create_groups.extend([instance_create_group.GROUP,
-                               instance_delete_group.GROUP])
+instance_create_groups.extend([groups.INST_CREATE,
+                               groups.INST_DELETE_WAIT])
 
 instance_error_create_groups = list(base_groups)
 instance_error_create_groups.extend([instance_error_create_group.GROUP])
 
-instance_upgrade_groups = list(instance_create_groups)
-instance_upgrade_groups.extend([instance_upgrade_group.GROUP])
-
 instance_force_delete_groups = list(base_groups)
 instance_force_delete_groups.extend([instance_force_delete_group.GROUP])
 
+instance_init_groups = list(base_groups)
+instance_init_groups.extend([instance_create_group.GROUP,
+                             instance_delete_group.GROUP])
+
+instance_upgrade_groups = list(instance_create_groups)
+instance_upgrade_groups.extend([instance_upgrade_group.GROUP])
+
 backup_groups = list(instance_create_groups)
 backup_groups.extend([groups.BACKUP,
                       groups.BACKUP_INST])
@@ -214,7 +218,7 @@ user_actions_groups.extend([user_actions_group.GROUP])
 
 # groups common to all datastores
 common_groups = list(instance_groups)
-common_groups.extend([guest_log_groups, module_groups])
+common_groups.extend([guest_log_groups, instance_init_groups, module_groups])
 
 # Register: Component based groups
 register(["backup"], backup_groups)
@@ -230,6 +234,7 @@ register(["instance_actions"], instance_actions_groups)
 register(["instance_create"], instance_create_groups)
 register(["instance_error"], instance_error_create_groups)
 register(["instance_force_delete"], instance_force_delete_groups)
+register(["instance_init"], instance_init_groups)
 register(["instance_upgrade"], instance_upgrade_groups)
 register(["module"], module_groups)
 register(["module_create"], module_create_groups)
diff --git a/trove/tests/scenario/groups/__init__.py b/trove/tests/scenario/groups/__init__.py
index 42cb2755..49de3a67 100644
--- a/trove/tests/scenario/groups/__init__.py
+++ b/trove/tests/scenario/groups/__init__.py
@@ -71,6 +71,8 @@ INST_UPGRADE = "scenario.inst_upgrade_grp"
 # Instance Create Group
 INST_CREATE = "scenario.inst_create_grp"
 INST_CREATE_WAIT = "scenario.inst_create_wait_grp"
+INST_INIT_CREATE = "scenario.inst_init_create_grp"
+INST_INIT_CREATE_WAIT = "scenario.inst_init_create_wait_grp"
 INST_INIT_DELETE = "scenario.inst_init_delete_grp"
 INST_INIT_DELETE_WAIT = "scenario.inst_init_delete_wait_grp"
 
diff --git a/trove/tests/scenario/groups/database_actions_group.py b/trove/tests/scenario/groups/database_actions_group.py
index ccd2fa61..b3b77c4a 100644
--- a/trove/tests/scenario/groups/database_actions_group.py
+++ b/trove/tests/scenario/groups/database_actions_group.py
@@ -136,17 +136,17 @@ class DatabaseActionsInstCreateWaitGroup(TestGroup):
 
     @test
     def wait_for_instances(self):
-        """Waiting for all instances to become active."""
-        self.instance_create_runner.run_wait_for_created_instances()
+        """Waiting for database instance to become active."""
+        self.instance_create_runner.run_wait_for_init_instance()
 
     @test(depends_on=[wait_for_instances])
     def add_initialized_instance_data(self):
-        """Add data to the initialized instance."""
+        """Add data to the database instance."""
         self.instance_create_runner.run_add_initialized_instance_data()
 
     @test(runs_after=[add_initialized_instance_data])
     def validate_initialized_instance(self):
-        """Validate the initialized instance data and properties."""
+        """Validate the database instance data and properties."""
         self.instance_create_runner.run_validate_initialized_instance()
 
 
@@ -162,7 +162,7 @@ class DatabaseActionsInstDeleteGroup(TestGroup):
 
     @test
     def delete_initialized_instance(self):
-        """Delete the initialized instance."""
+        """Delete the database instance."""
         self.instance_create_runner.run_initialized_instance_delete()
 
 
@@ -179,5 +179,5 @@ class DatabaseActionsInstDeleteWaitGroup(TestGroup):
 
     @test
     def wait_for_delete_initialized_instance(self):
-        """Wait for the initialized instance to delete."""
+        """Wait for the database instance to delete."""
         self.instance_create_runner.run_wait_for_init_delete()
diff --git a/trove/tests/scenario/groups/instance_create_group.py b/trove/tests/scenario/groups/instance_create_group.py
index 228be246..83a1f16b 100644
--- a/trove/tests/scenario/groups/instance_create_group.py
+++ b/trove/tests/scenario/groups/instance_create_group.py
@@ -45,7 +45,17 @@ class InstanceCreateGroup(TestGroup):
         """Create an empty instance."""
         self.test_runner.run_empty_instance_create()
 
-    @test(runs_after=[create_empty_instance])
+
+@test(depends_on_groups=[groups.INST_CREATE],
+      groups=[GROUP, groups.INST_INIT_CREATE])
+class InstanceInitCreateGroup(TestGroup):
+    """Test Instance Init Create functionality."""
+
+    def __init__(self):
+        super(InstanceInitCreateGroup, self).__init__(
+            InstanceCreateRunnerFactory.instance())
+
+    @test
     def create_initial_configuration(self):
         """Create a configuration group for a new initialized instance."""
         self.test_runner.run_initial_configuration_create()
@@ -59,7 +69,7 @@ class InstanceCreateGroup(TestGroup):
 @test(depends_on_groups=[groups.INST_CREATE],
       groups=[GROUP, groups.INST_CREATE_WAIT],
       runs_after_groups=[groups.MODULE_CREATE, groups.CFGGRP_CREATE,
-                         groups.INST_ERROR_CREATE_WAIT])
+                         groups.INST_ERROR_DELETE])
 class InstanceCreateWaitGroup(TestGroup):
     """Test that Instance Create Completes."""
 
@@ -68,11 +78,27 @@ class InstanceCreateWaitGroup(TestGroup):
             InstanceCreateRunnerFactory.instance())
 
     @test
-    def wait_for_instances(self):
-        """Waiting for all instances to become active."""
-        self.test_runner.run_wait_for_created_instances()
+    def wait_for_instance(self):
+        """Waiting for main instance to become active."""
+        self.test_runner.run_wait_for_instance()
+
+
+@test(depends_on_groups=[groups.INST_INIT_CREATE],
+      groups=[GROUP, groups.INST_INIT_CREATE_WAIT],
+      runs_after_groups=[groups.INST_CREATE_WAIT])
+class InstanceInitCreateWaitGroup(TestGroup):
+    """Test that Instance Init Create Completes."""
+
+    def __init__(self):
+        super(InstanceInitCreateWaitGroup, self).__init__(
+            InstanceCreateRunnerFactory.instance())
+
+    @test
+    def wait_for_init_instance(self):
+        """Waiting for init instance to become active."""
+        self.test_runner.run_wait_for_init_instance()
 
-    @test(depends_on=[wait_for_instances])
+    @test(depends_on=[wait_for_init_instance])
     def add_initialized_instance_data(self):
         """Add data to the initialized instance."""
         self.test_runner.run_add_initialized_instance_data()
@@ -83,7 +109,7 @@ class InstanceCreateWaitGroup(TestGroup):
         self.test_runner.run_validate_initialized_instance()
 
 
-@test(depends_on_groups=[groups.INST_CREATE_WAIT],
+@test(depends_on_groups=[groups.INST_INIT_CREATE_WAIT],
       groups=[GROUP, groups.INST_INIT_DELETE])
 class InstanceInitDeleteGroup(TestGroup):
     """Test Initialized Instance Delete functionality."""
diff --git a/trove/tests/scenario/groups/user_actions_group.py b/trove/tests/scenario/groups/user_actions_group.py
index db13e72c..b3ae309d 100644
--- a/trove/tests/scenario/groups/user_actions_group.py
+++ b/trove/tests/scenario/groups/user_actions_group.py
@@ -229,12 +229,12 @@ class UserActionsInstCreateWaitGroup(TestGroup):
 
     @test
     def wait_for_instances(self):
-        """Waiting for all instances to become active."""
-        self.instance_create_runner.run_wait_for_created_instances()
+        """Waiting for user instance to become active."""
+        self.instance_create_runner.run_wait_for_init_instance()
 
     @test(depends_on=[wait_for_instances])
     def validate_initialized_instance(self):
-        """Validate the initialized instance data and properties."""
+        """Validate the user instance data and properties."""
         self.instance_create_runner.run_validate_initialized_instance()
 
 
@@ -250,7 +250,7 @@ class UserActionsInstDeleteGroup(TestGroup):
 
     @test
     def delete_initialized_instance(self):
-        """Delete the initialized instance."""
+        """Delete the user instance."""
         self.instance_create_runner.run_initialized_instance_delete()
 
 
@@ -268,5 +268,5 @@ class UserActionsInstDeleteWaitGroup(TestGroup):
 
     @test
     def wait_for_delete_initialized_instance(self):
-        """Wait for the initialized instance to delete."""
+        """Wait for the user instance to delete."""
         self.instance_create_runner.run_wait_for_init_delete()
diff --git a/trove/tests/scenario/runners/backup_runners.py b/trove/tests/scenario/runners/backup_runners.py
index a17a4c79..5e660aab 100644
--- a/trove/tests/scenario/runners/backup_runners.py
+++ b/trove/tests/scenario/runners/backup_runners.py
@@ -111,11 +111,15 @@ class BackupRunner(TestRunner):
 
     def assert_backup_create(self, name, desc, instance_id, parent_id=None,
                              incremental=False):
+        client = self.auth_client
+        datastore_version = client.datastore_versions.get(
+            self.instance_info.dbaas_datastore,
+            self.instance_info.dbaas_datastore_version)
         if incremental:
-            result = self.auth_client.backups.create(
+            result = client.backups.create(
                 name, instance_id, desc, incremental=incremental)
         else:
-            result = self.auth_client.backups.create(
+            result = client.backups.create(
                 name, instance_id, desc, parent_id=parent_id)
         self.assert_equal(name, result.name,
                           'Unexpected backup name')
@@ -129,11 +133,7 @@ class BackupRunner(TestRunner):
             self.assert_equal(parent_id, result.parent_id,
                               'Unexpected status for backup')
 
-        instance = self.auth_client.instances.get(instance_id)
-        datastore_version = self.auth_client.datastore_versions.get(
-            self.instance_info.dbaas_datastore,
-            self.instance_info.dbaas_datastore_version)
-
+        instance = client.instances.get(instance_id)
         self.assert_equal('BACKUP', instance.status,
                           'Unexpected instance status')
         self.assert_equal(self.instance_info.dbaas_datastore,
diff --git a/trove/tests/scenario/runners/instance_create_runners.py b/trove/tests/scenario/runners/instance_create_runners.py
index b7ee401f..879197bc 100644
--- a/trove/tests/scenario/runners/instance_create_runners.py
+++ b/trove/tests/scenario/runners/instance_create_runners.py
@@ -58,8 +58,6 @@ class InstanceCreateRunner(TestRunner):
             instance_info.dbaas_datastore_version)
         self.instance_info.dbaas_flavor_href = instance_info.dbaas_flavor_href
         self.instance_info.volume = instance_info.volume
-        self.instance_info.srv_grp_id = self.assert_server_group_exists(
-            self.instance_info.id)
         self.instance_info.helper_user = instance_info.helper_user
         self.instance_info.helper_database = instance_info.helper_database
 
@@ -230,18 +228,19 @@ class InstanceCreateRunner(TestRunner):
 
         return instance_info
 
-    def run_wait_for_created_instances(
+    def run_wait_for_instance(
             self, expected_states=['BUILD', 'ACTIVE']):
         instances = [self.instance_info.id]
-        if self.init_inst_info:
-            instances.append(self.init_inst_info.id)
         self.assert_all_instance_states(instances, expected_states)
-
-        # It may take some time before the initial users/databases
-        # appear on the instance after it went ACTIVE.
-        # We need to wait for that before we can proceed with data operations.
+        self.instance_info.srv_grp_id = self.assert_server_group_exists(
+            self.instance_info.id)
         self.wait_for_test_helpers(self.instance_info)
+
+    def run_wait_for_init_instance(
+            self, expected_states=['BUILD', 'ACTIVE']):
         if self.init_inst_info:
+            instances = [self.init_inst_info.id]
+            self.assert_all_instance_states(instances, expected_states)
             self.wait_for_test_helpers(self.init_inst_info)
 
     def wait_for_test_helpers(self, inst_info):
diff --git a/trove/tests/scenario/runners/test_runners.py b/trove/tests/scenario/runners/test_runners.py
index 7904ffed..018ed418 100644
--- a/trove/tests/scenario/runners/test_runners.py
+++ b/trove/tests/scenario/runners/test_runners.py
@@ -493,16 +493,18 @@ class TestRunner(object):
             fast_fail_status = ['ERROR', 'FAILED']
         found = False
         for status in expected_states:
-            if require_all_states or found or self._has_status(
-                    instance_id, status, fast_fail_status=fast_fail_status):
+            found_current = self._has_status(
+                instance_id, status, fast_fail_status=fast_fail_status)
+            if require_all_states or found or found_current:
                 found = True
                 start_time = timer.time()
                 try:
-                    poll_until(lambda: self._has_status(
-                        instance_id, status,
-                        fast_fail_status=fast_fail_status),
-                        sleep_time=self.def_sleep_time,
-                        time_out=self.def_timeout)
+                    if not found_current:
+                        poll_until(lambda: self._has_status(
+                            instance_id, status,
+                            fast_fail_status=fast_fail_status),
+                            sleep_time=self.def_sleep_time,
+                            time_out=self.def_timeout)
                     self.report.log("Instance '%s' has gone '%s' in %s." %
                                     (instance_id, status,
                                      self._time_since(start_time)))