Support instance operating_status

Added a new field ``operating_status`` for the instance to show the actual
operational status of user's database.

Change-Id: I7c52cff0ec48289fe1a260e99e02a506d4f8ddec

4 files changed, 100 insertions, 63 deletions

diff --git a/trove/tests/api/instances.py b/trove/tests/api/instances.py
index 643e120e..7f0f7887 100644
--- a/trove/tests/api/instances.py
+++ b/trove/tests/api/instances.py
@@ -622,7 +622,7 @@ class CreateInstanceFail(object):
                              'server_state_description', 'status', 'updated',
                              'users', 'volume', 'root_enabled_at',
                              'root_enabled_by', 'fault',
-                             'service_status_updated']
+                             'service_status_updated', 'operating_status']
             with CheckInstance(result._info) as check:
                 check.contains_allowed_attrs(
                     result._info, allowed_attrs,
@@ -785,7 +785,8 @@ class CreateInstance(object):
         # Check these attrs only are returned in create response
         allowed_attrs = ['created', 'flavor', 'addresses', 'id', 'links',
                          'name', 'status', 'updated', 'datastore', 'fault',
-                         'region', 'service_status_updated', 'access']
+                         'region', 'service_status_updated', 'access',
+                         'operating_status']
         if ROOT_ON_CREATE:
             allowed_attrs.append('password')
         if VOLUME_SUPPORT:
@@ -928,7 +929,7 @@ class TestGetInstances(object):
     def test_index_list(self):
         allowed_attrs = ['id', 'links', 'name', 'status', 'flavor',
                          'datastore', 'ip', 'hostname', 'replica_of',
-                         'region', 'addresses', 'access']
+                         'region', 'addresses', 'access', 'operating_status']
         if VOLUME_SUPPORT:
             allowed_attrs.append('volume')
         instances = dbaas.instances.list()
@@ -950,7 +951,8 @@ class TestGetInstances(object):
         allowed_attrs = ['created', 'databases', 'flavor', 'hostname', 'id',
                          'links', 'name', 'status', 'updated', 'ip',
                          'datastore', 'fault', 'region',
-                         'service_status_updated', 'addresses', 'access']
+                         'service_status_updated', 'addresses', 'access',
+                         'operating_status']
         if VOLUME_SUPPORT:
             allowed_attrs.append('volume')
         instances = dbaas.instances.list(detailed=True)
@@ -970,7 +972,8 @@ class TestGetInstances(object):
         allowed_attrs = ['created', 'databases', 'flavor', 'hostname', 'id',
                          'links', 'name', 'status', 'updated', 'ip',
                          'datastore', 'fault', 'region',
-                         'service_status_updated', 'addresses', 'access']
+                         'service_status_updated', 'addresses', 'access',
+                         'operating_status']
         if VOLUME_SUPPORT:
             allowed_attrs.append('volume')
         else:
@@ -1050,7 +1053,7 @@ class TestGetInstances(object):
                          'name', 'root_enabled_at', 'root_enabled_by',
                          'server_state_description', 'status', 'datastore',
                          'updated', 'users', 'volume', 'fault', 'region',
-                         'access']
+                         'access', 'operating_status']
         with CheckInstance(result._info) as check:
             check.contains_allowed_attrs(
                 result._info, allowed_attrs,
diff --git a/trove/tests/api/instances_actions.py b/trove/tests/api/instances_actions.py
index 4d8d2b25..d35d0f7c 100644
--- a/trove/tests/api/instances_actions.py
+++ b/trove/tests/api/instances_actions.py
@@ -276,6 +276,15 @@ class RebootTestBase(ActionTestBase):
 
         poll_until(is_status, time_out=timeout, sleep_time=sleep_time)
 
+    def wait_for_operating_status(self, status, timeout=60, sleep_time=5):
+        def is_status():
+            instance = self.instance
+            if instance.operating_status in status:
+                return True
+            return False
+
+        poll_until(is_status, time_out=timeout, sleep_time=sleep_time)
+
 
 @test(groups=[tests.DBAAS_API_INSTANCE_ACTIONS],
       depends_on_groups=[tests.DBAAS_API_DATABASES],
@@ -323,9 +332,7 @@ class StopTests(RebootTestBase):
     def test_stop_mysql(self):
         """Stops MySQL by admin."""
         instance_info.dbaas_admin.management.stop(self.instance_id)
-
-        # The instance status will only be updated by guest agent.
-        self.wait_for_status(['SHUTDOWN'], timeout=90, sleep_time=10)
+        self.wait_for_operating_status(['SHUTDOWN'], timeout=90, sleep_time=10)
 
     @test(depends_on=[test_stop_mysql])
     def test_volume_info_while_mysql_is_down(self):
diff --git a/trove/tests/scenario/runners/test_runners.py b/trove/tests/scenario/runners/test_runners.py
index 60326773..d1dacd0c 100644
--- a/trove/tests/scenario/runners/test_runners.py
+++ b/trove/tests/scenario/runners/test_runners.py
@@ -325,7 +325,7 @@ class TestRunner(object, metaclass=LogOnFail):
     instance_info = InstanceTestInfo()
     report = CONFIG.get_report()
 
-    def __init__(self, sleep_time=10, timeout=1800):
+    def __init__(self, sleep_time=10, timeout=900):
         self.def_sleep_time = sleep_time
         self.def_timeout = timeout
 
@@ -604,6 +604,19 @@ class TestRunner(object, metaclass=LogOnFail):
             self.assert_equal(expected_http_code, client.last_http_code,
                               "Unexpected client status code")
 
+    def assert_instance_operating_status(self, instance_id, expected_status):
+        self.report.log(f"Waiting for expected_status ({expected_status}) "
+                        f"for instances: {instance_id}")
+
+        def wait_for_operating_status():
+            instance = self.get_instance(instance_id, self.admin_client)
+            if instance.operating_status == expected_status:
+                return True
+            return False
+
+        poll_until(wait_for_operating_status, sleep_time=self.def_sleep_time,
+                   time_out=self.def_timeout)
+
     def assert_all_instance_states(self, instance_ids, expected_states,
                                    fast_fail_status=None,
                                    require_all_states=False):
@@ -647,6 +660,15 @@ class TestRunner(object, metaclass=LogOnFail):
         self.report.log("Waiting for states (%s) for instance: %s" %
                         (expected_states, instance_id))
 
+        # Replace HEALTHY with ACTIVE. This is needed after operating_status
+        # is introduced in Trove.
+        wait_operating_status = False
+        if 'HEALTHY' in expected_states:
+            wait_operating_status = True
+            expected_states.remove('HEALTHY')
+            if 'ACTIVE' not in expected_states:
+                expected_states.append('ACTIVE')
+
         if fast_fail_status is None:
             fast_fail_status = ['ERROR', 'FAILED']
         found = False
@@ -677,6 +699,9 @@ class TestRunner(object, metaclass=LogOnFail):
                     "Instance state was not '%s', moving to the next expected "
                     "state." % status)
 
+        if found and wait_operating_status:
+            self.assert_instance_operating_status(instance_id, 'HEALTHY')
+
         return found
 
     def _time_since(self, start_time):
@@ -992,7 +1017,7 @@ class CheckInstance(AttrCheck):
         if 'datastore' not in self.instance:
             self.fail("'datastore' not found in instance.")
         else:
-            allowed_attrs = ['type', 'version']
+            allowed_attrs = ['type', 'version', 'version_number']
             self.contains_allowed_attrs(
                 self.instance['datastore'], allowed_attrs,
                 msg="datastore")
diff --git a/trove/tests/unittests/instance/test_instance_status.py b/trove/tests/unittests/instance/test_instance_status.py
index a81849ce..aaa8bcd4 100644
--- a/trove/tests/unittests/instance/test_instance_status.py
+++ b/trove/tests/unittests/instance/test_instance_status.py
@@ -13,6 +13,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 #
+from unittest import mock
 import uuid
 
 from trove.datastore import models
@@ -38,113 +39,114 @@ class FakeDBInstance(object):
         self.id = str(uuid.uuid4())
         self.deleted = False
         self.datastore_version_id = str(uuid.uuid4())
-        self.server_status = "HEALTHY"
+        self.server_status = "ACTIVE"
         self.task_status = FakeInstanceTask()
 
 
 class BaseInstanceStatusTestCase(trove_testtools.TestCase):
-
-    def setUp(self):
+    @classmethod
+    def setUpClass(cls):
         util.init_db()
-        self.db_info = FakeDBInstance()
-        self.status = InstanceServiceStatus(
-            ServiceStatuses.RUNNING)
-        self.datastore = models.DBDatastore.create(
+        cls.db_info = FakeDBInstance()
+        cls.datastore = models.DBDatastore.create(
             id=str(uuid.uuid4()),
             name='mysql' + str(uuid.uuid4()),
-            default_version_id=self.db_info.datastore_version_id
+            default_version_id=cls.db_info.datastore_version_id
         )
-        self.version = models.DBDatastoreVersion.create(
-            id=self.db_info.datastore_version_id,
-            datastore_id=self.datastore.id,
+        cls.version = models.DBDatastoreVersion.create(
+            id=cls.db_info.datastore_version_id,
+            datastore_id=cls.datastore.id,
             name='5.7' + str(uuid.uuid4()),
             manager='mysql',
             image_id=str(uuid.uuid4()),
             active=1,
             packages="mysql-server-5.7"
         )
-        super(BaseInstanceStatusTestCase, self).setUp()
+        super(BaseInstanceStatusTestCase, cls).setUpClass()
 
-    def tearDown(self):
-        self.datastore.delete()
-        self.version.delete()
-        super(BaseInstanceStatusTestCase, self).tearDown()
+    @classmethod
+    def tearDownClass(cls):
+        util.cleanup_db()
+        super(BaseInstanceStatusTestCase, cls).tearDownClass()
 
 
 class InstanceStatusTest(BaseInstanceStatusTestCase):
+    def setUp(self):
+        self.db_info.task_status = FakeInstanceTask()
+        self.db_info.server_status = "ACTIVE"
+        self.ds_status = InstanceServiceStatus(ServiceStatuses.HEALTHY)
+        super(InstanceStatusTest, self).setUp()
 
     def test_task_status_error_reports_error(self):
         self.db_info.task_status.is_error = True
-        instance = SimpleInstance('dummy context', self.db_info, self.status)
+        instance = SimpleInstance('dummy context', self.db_info,
+                                  self.ds_status)
         self.assertEqual(InstanceStatus.ERROR, instance.status)
 
     def test_task_status_action_building_reports_build(self):
         self.db_info.task_status.action = "BUILDING"
-        instance = SimpleInstance('dummy context', self.db_info, self.status)
+        instance = SimpleInstance('dummy context', self.db_info,
+                                  self.ds_status)
         self.assertEqual(InstanceStatus.BUILD, instance.status)
 
     def test_task_status_action_rebooting_reports_reboot(self):
         self.db_info.task_status.action = "REBOOTING"
-        instance = SimpleInstance('dummy context', self.db_info, self.status)
+        instance = SimpleInstance('dummy context', self.db_info,
+                                  self.ds_status)
         self.assertEqual(InstanceStatus.REBOOT, instance.status)
 
     def test_task_status_action_resizing_reports_resize(self):
         self.db_info.task_status.action = "RESIZING"
-        instance = SimpleInstance('dummy context', self.db_info, self.status)
+        instance = SimpleInstance('dummy context', self.db_info,
+                                  self.ds_status)
         self.assertEqual(InstanceStatus.RESIZE, instance.status)
 
-    def test_task_status_action_deleting_reports_shutdown(self):
+    def test_task_deleting_server_active(self):
         self.db_info.task_status.action = "DELETING"
-        instance = SimpleInstance('dummy context', self.db_info, self.status)
+        instance = SimpleInstance('dummy context', self.db_info,
+                                  self.ds_status)
         self.assertEqual(InstanceStatus.SHUTDOWN, instance.status)
 
     def test_nova_server_build_reports_build(self):
         self.db_info.server_status = "BUILD"
-        instance = SimpleInstance('dummy context', self.db_info, self.status)
+        instance = SimpleInstance('dummy context', self.db_info,
+                                  self.ds_status)
         self.assertEqual(InstanceStatus.BUILD, instance.status)
 
     def test_nova_server_error_reports_error(self):
         self.db_info.server_status = "ERROR"
-        instance = SimpleInstance('dummy context', self.db_info, self.status)
+        instance = SimpleInstance('dummy context', self.db_info,
+                                  self.ds_status)
         self.assertEqual(InstanceStatus.ERROR, instance.status)
 
     def test_nova_server_reboot_reports_reboot(self):
         self.db_info.server_status = "REBOOT"
-        instance = SimpleInstance('dummy context', self.db_info, self.status)
+        instance = SimpleInstance('dummy context', self.db_info,
+                                  self.ds_status)
         self.assertEqual(InstanceStatus.REBOOT, instance.status)
 
     def test_nova_server_resize_reports_resize(self):
         self.db_info.server_status = "RESIZE"
-        instance = SimpleInstance('dummy context', self.db_info, self.status)
+        instance = SimpleInstance('dummy context', self.db_info,
+                                  self.ds_status)
         self.assertEqual(InstanceStatus.RESIZE, instance.status)
 
     def test_nova_server_verify_resize_reports_resize(self):
         self.db_info.server_status = "VERIFY_RESIZE"
-        instance = SimpleInstance('dummy context', self.db_info, self.status)
+        instance = SimpleInstance('dummy context', self.db_info,
+                                  self.ds_status)
         self.assertEqual(InstanceStatus.RESIZE, instance.status)
 
-    def test_service_status_paused_reports_reboot(self):
-        self.status.set_status(ServiceStatuses.PAUSED)
-        instance = SimpleInstance('dummy context', self.db_info, self.status)
-        self.assertEqual(InstanceStatus.REBOOT, instance.status)
-
-    def test_service_status_new_reports_build(self):
-        self.status.set_status(ServiceStatuses.NEW)
-        instance = SimpleInstance('dummy context', self.db_info, self.status)
-        self.assertEqual(InstanceStatus.BUILD, instance.status)
-
-    def test_service_status_running_reports_active(self):
-        self.status.set_status(ServiceStatuses.RUNNING)
-        instance = SimpleInstance('dummy context', self.db_info, self.status)
-        self.assertEqual(InstanceStatus.ACTIVE, instance.status)
-
-    def test_service_status_reset_status(self):
-        self.status.set_status(ServiceStatuses.UNKNOWN)
-        instance = SimpleInstance('dummy context', self.db_info, self.status)
-        self.assertEqual(InstanceStatus.ERROR, instance.status)
-
-    def test_service_status_force_deleteing(self):
-        self.status.set_status(ServiceStatuses.UNKNOWN)
-        self.db_info.task_status = InstanceTasks.DELETING
-        instance = SimpleInstance('dummy context', self.db_info, self.status)
-        self.assertEqual(InstanceStatus.SHUTDOWN, instance.status)
+    def test_operating_status_healthy(self):
+        self.db_info.task_status = InstanceTasks.NONE
+        instance = SimpleInstance(mock.MagicMock(), self.db_info,
+                                  self.ds_status)
+        self.assertEqual(repr(ServiceStatuses.HEALTHY),
+                         instance.operating_status)
+
+    def test_operating_status_task_not_none(self):
+        self.db_info.task_status = InstanceTasks.RESIZING
+        instance = SimpleInstance(mock.MagicMock(), self.db_info,
+                                  self.ds_status)
+        self.assertEqual("",
+                         instance.operating_status)