Merge "Implement Instance Upgrade"

28 files changed, 504 insertions, 56 deletions

diff --git a/releasenotes/notes/instance-upgrade-7d464f85e025d729.yaml b/releasenotes/notes/instance-upgrade-7d464f85e025d729.yaml
new file mode 100644
index 00000000..0075b567
--- /dev/null
+++ b/releasenotes/notes/instance-upgrade-7d464f85e025d729.yaml
@@ -0,0 +1,4 @@
+features:
+  - New instance upgrade API supports upgrading an instance of
+    a datastore to a new datastore version.  Includes implementation
+    for MySQL family of databases.
diff --git a/trove/common/apischema.py b/trove/common/apischema.py
index 5752c0ae..90e86b43 100644
--- a/trove/common/apischema.py
+++ b/trove/common/apischema.py
@@ -370,6 +370,7 @@ instance = {
                     "replica_of": {},
                     "name": non_empty_string,
                     "configuration": configuration_id,
+                    "datastore_version": non_empty_string,
                 }
             }
         }
diff --git a/trove/common/notification.py b/trove/common/notification.py
index ee702adc..5bf23ab3 100644
--- a/trove/common/notification.py
+++ b/trove/common/notification.py
@@ -347,6 +347,10 @@ class DBaaSAPINotification(object):
     def server_type(self, server_type):
         self.payload['server_type'] = server_type
 
+    @property
+    def request_id(self):
+        return self.payload['request_id']
+
     def __init__(self, context, **kwargs):
         self.context = context
         self.needs_end_notification = True
@@ -753,3 +757,14 @@ class DBaaSConfigurationEdit(DBaaSAPINotification):
     @abc.abstractmethod
     def required_start_traits(self):
         return ['configuration_id']
+
+
+class DBaaSInstanceUpgrade(DBaaSAPINotification):
+
+    @abc.abstractmethod
+    def event_type(self):
+        return 'upgrade'
+
+    @abc.abstractmethod
+    def required_start_traits(self):
+        return ['instance_id', 'datastore_version_id']
diff --git a/trove/conductor/manager.py b/trove/conductor/manager.py
index 1d0e7f89..79a33f2c 100644
--- a/trove/conductor/manager.py
+++ b/trove/conductor/manager.py
@@ -149,4 +149,6 @@ class Manager(periodic_task.PeriodicTasks):
                         message, exception):
         notification = SerializableNotification.deserialize(
             context, serialized_notification)
+        LOG.error(_("Guest exception on request %(req)s:\n%(exc)s")
+                  % {'req': notification.request_id, 'exc': exception})
         notification.notify_exc_info(message, exception)
diff --git a/trove/datastore/models.py b/trove/datastore/models.py
index a769d8d7..cd67019d 100644
--- a/trove/datastore/models.py
+++ b/trove/datastore/models.py
@@ -336,6 +336,9 @@ class Datastore(object):
     def __init__(self, db_info):
         self.db_info = db_info
 
+    def __repr__(self, *args, **kwargs):
+        return "%s(%s)" % (self.name, self.id)
+
     @classmethod
     def load(cls, id_or_name):
         try:
@@ -387,6 +390,9 @@ class DatastoreVersion(object):
         self.db_info = db_info
         self._datastore_name = None
 
+    def __repr__(self, *args, **kwargs):
+        return "%s(%s)" % (self.name, self.id)
+
     @classmethod
     def load(cls, datastore, id_or_name):
         try:
diff --git a/trove/guestagent/api.py b/trove/guestagent/api.py
index 77e62cba..43e46f6b 100644
--- a/trove/guestagent/api.py
+++ b/trove/guestagent/api.py
@@ -267,6 +267,17 @@ class API(object):
                 server.stop()
                 server.wait()
 
+    def pre_upgrade(self):
+        """Prepare the guest for upgrade."""
+        LOG.debug("Sending the call to prepare the guest for upgrade.")
+        return self._call("pre_upgrade", AGENT_HIGH_TIMEOUT, self.version_cap)
+
+    def post_upgrade(self, upgrade_info):
+        """Recover the guest after upgrading the guest's image."""
+        LOG.debug("Recover the guest after upgrading the guest's image.")
+        self._call("post_upgrade", AGENT_HIGH_TIMEOUT, self.version_cap,
+                   upgrade_info=upgrade_info)
+
     def restart(self):
         """Restart the database server."""
         LOG.debug("Sending the call to restart the database process "
diff --git a/trove/guestagent/common/configuration.py b/trove/guestagent/common/configuration.py
index 0e52bc0b..1fd226d2 100644
--- a/trove/guestagent/common/configuration.py
+++ b/trove/guestagent/common/configuration.py
@@ -96,7 +96,7 @@ class ConfigurationManager(object):
         """Return the current value at a given key or 'default'.
         """
         if self._value_cache is None:
-            self._refresh_cache()
+            self.refresh_cache()
 
         return self._value_cache.get(key, default)
 
@@ -139,7 +139,7 @@ class ConfigurationManager(object):
                 self._base_config_path, FileMode.ADD_READ_ALL,
                 as_root=self._requires_root)
 
-            self._refresh_cache()
+            self.refresh_cache()
 
     def has_system_override(self, change_id):
         """Return whether a given 'system' change exists.
@@ -178,7 +178,7 @@ class ConfigurationManager(object):
                 group_name, change_id, self._codec.deserialize(options))
         else:
             self._override_strategy.apply(group_name, change_id, options)
-            self._refresh_cache()
+            self.refresh_cache()
 
     def remove_system_override(self, change_id=DEFAULT_CHANGE_ID):
         """Revert a 'system' configuration change.
@@ -192,9 +192,9 @@ class ConfigurationManager(object):
 
     def _remove_override(self, group_name, change_id):
         self._override_strategy.remove(group_name, change_id)
-        self._refresh_cache()
+        self.refresh_cache()
 
-    def _refresh_cache(self):
+    def refresh_cache(self):
         self._value_cache = self.parse_configuration()
 
 
diff --git a/trove/guestagent/datastore/manager.py b/trove/guestagent/datastore/manager.py
index 36273212..87371c37 100644
--- a/trove/guestagent/datastore/manager.py
+++ b/trove/guestagent/datastore/manager.py
@@ -252,7 +252,7 @@ class Manager(periodic_task.PeriodicTasks):
         return True
 
     #################
-    # Prepare related
+    # Instance related
     #################
     def prepare(self, context, packages, databases, memory_mb, users,
                 device_path=None, mount_point=None, backup_info=None,
@@ -389,6 +389,18 @@ class Manager(periodic_task.PeriodicTasks):
         LOG.info(_('No post_prepare work has been defined.'))
         pass
 
+    def pre_upgrade(self, context):
+        """Prepares the guest for upgrade, returning a dict to be passed
+        to post_upgrade
+        """
+        return {}
+
+    def post_upgrade(self, context, upgrade_info):
+        """Recovers the guest after the image is upgraded using infomation
+        from the pre_upgrade step
+        """
+        pass
+
     #################
     # Service related
     #################
@@ -407,11 +419,12 @@ class Manager(periodic_task.PeriodicTasks):
         LOG.debug("Getting file system stats for '%s'" % mount_point)
         return dbaas.get_filesystem_volume_stats(mount_point)
 
-    def mount_volume(self, context, device_path=None, mount_point=None):
+    def mount_volume(self, context, device_path=None, mount_point=None,
+                     write_to_fstab=False):
         LOG.debug("Mounting the device %s at the mount point %s." %
                   (device_path, mount_point))
         device = volume.VolumeDevice(device_path)
-        device.mount(mount_point, write_to_fstab=False)
+        device.mount(mount_point, write_to_fstab=write_to_fstab)
 
     def unmount_volume(self, context, device_path=None, mount_point=None):
         LOG.debug("Unmounting the device %s from the mount point %s." %
diff --git a/trove/guestagent/datastore/mysql_common/manager.py b/trove/guestagent/datastore/mysql_common/manager.py
index 9a8c613f..b670dd0f 100644
--- a/trove/guestagent/datastore/mysql_common/manager.py
+++ b/trove/guestagent/datastore/mysql_common/manager.py
@@ -242,6 +242,56 @@ class MySqlManager(manager.Manager):
         if snapshot:
             self.attach_replica(context, snapshot, snapshot['config'])
 
+    def pre_upgrade(self, context):
+        app = self.mysql_app(self.mysql_app_status.get())
+        data_dir = app.get_data_dir()
+        mount_point, _data = os.path.split(data_dir)
+        save_dir = "%s/etc_mysql" % mount_point
+        save_etc_dir = "%s/etc" % mount_point
+        home_save = "%s/trove_user" % mount_point
+
+        app.status.begin_restart()
+        app.stop_db()
+
+        if operating_system.exists("/etc/my.cnf", as_root=True):
+            operating_system.create_directory(save_etc_dir, as_root=True)
+            operating_system.copy("/etc/my.cnf", save_etc_dir,
+                                  preserve=True, as_root=True)
+
+        operating_system.copy("/etc/mysql/.", save_dir,
+                              preserve=True, as_root=True)
+
+        operating_system.copy("%s/." % os.path.expanduser('~'), home_save,
+                              preserve=True, as_root=True)
+
+        self.unmount_volume(context, mount_point=data_dir)
+        return {
+            'mount_point': mount_point,
+            'save_dir': save_dir,
+            'save_etc_dir': save_etc_dir,
+            'home_save': home_save
+        }
+
+    def post_upgrade(self, context, upgrade_info):
+        app = self.mysql_app(self.mysql_app_status.get())
+        app.stop_db()
+        if 'device' in upgrade_info:
+            self.mount_volume(context, mount_point=upgrade_info['mount_point'],
+                              device_path=upgrade_info['device'],
+                              write_to_fstab=True)
+
+        if operating_system.exists(upgrade_info['save_etc_dir'],
+                                   is_directory=True, as_root=True):
+            operating_system.copy("%s/." % upgrade_info['save_etc_dir'],
+                                  "/etc", preserve=True, as_root=True)
+
+        operating_system.copy("%s/." % upgrade_info['save_dir'], "/etc/mysql",
+                              preserve=True, as_root=True)
+        operating_system.copy("%s/." % upgrade_info['home_save'],
+                              os.path.expanduser('~'),
+                              preserve=True, as_root=True)
+        app.start_mysql()
+
     def restart(self, context):
         app = self.mysql_app(self.mysql_app_status.get())
         app.restart()
diff --git a/trove/instance/models.py b/trove/instance/models.py
index 6ad3d5c5..6f213ed3 100644
--- a/trove/instance/models.py
+++ b/trove/instance/models.py
@@ -17,6 +17,7 @@
 """Model classes that form the core of instances functionality."""
 from datetime import datetime
 from datetime import timedelta
+import os.path
 import re
 
 from novaclient import exceptions as nova_exceptions
@@ -98,6 +99,7 @@ class InstanceStatus(object):
     RESTART_REQUIRED = "RESTART_REQUIRED"
     PROMOTE = "PROMOTE"
     EJECT = "EJECT"
+    UPGRADE = "UPGRADE"
     DETACH = "DETACH"
 
 
@@ -129,7 +131,8 @@ def load_simple_instance_server_status(context, db_info):
 
 
 # Invalid states to contact the agent
-AGENT_INVALID_STATUSES = ["BUILD", "REBOOT", "RESIZE", "PROMOTE", "EJECT"]
+AGENT_INVALID_STATUSES = ["BUILD", "REBOOT", "RESIZE", "PROMOTE", "EJECT",
+                          "UPGRADE"]
 
 
 class SimpleInstance(object):
@@ -175,6 +178,9 @@ class SimpleInstance(object):
 
         self.slave_list = None
 
+    def __repr__(self, *args, **kwargs):
+        return "%s(%s)" % (self.name, self.id)
+
     @property
     def addresses(self):
         # TODO(tim.simpson): This code attaches two parts of the Nova server to
@@ -296,6 +302,8 @@ class SimpleInstance(object):
             return InstanceStatus.REBOOT
         if 'RESIZING' == action:
             return InstanceStatus.RESIZE
+        if 'UPGRADING' == action:
+            return InstanceStatus.UPGRADE
         if 'RESTART_REQUIRED' == action:
             return InstanceStatus.RESTART_REQUIRED
         if InstanceTasks.PROMOTING.action == action:
@@ -684,6 +692,32 @@ class BaseInstance(SimpleInstance):
             self._server_group_loaded = True
         return self._server_group
 
+    def get_injected_files(self, datastore_manager):
+        injected_config_location = CONF.get('injected_config_location')
+        guest_info = CONF.get('guest_info')
+
+        if ('/' in guest_info):
+            # Set guest_info_file to exactly guest_info from the conf file.
+            # This should be /etc/guest_info for pre-Kilo compatibility.
+            guest_info_file = guest_info
+        else:
+            guest_info_file = os.path.join(injected_config_location,
+                                           guest_info)
+
+        files = {guest_info_file: (
+            "[DEFAULT]\n"
+            "guest_id=%s\n"
+            "datastore_manager=%s\n"
+            "tenant_id=%s\n"
+            % (self.id, datastore_manager, self.tenant_id))}
+
+        if os.path.isfile(CONF.get('guest_config')):
+            with open(CONF.get('guest_config'), "r") as f:
+                files[os.path.join(injected_config_location,
+                                   "trove-guestagent.conf")] = f.read()
+
+        return files
+
 
 class FreshInstance(BaseInstance):
     @classmethod
@@ -1230,6 +1264,12 @@ class Instance(BuiltInstance):
             self.datastore_version, flavor, self.id)
         return dict(config.render_dict())
 
+    def upgrade(self, datastore_version):
+        self.update_db(datastore_version_id=datastore_version.id,
+                       task_status=InstanceTasks.UPGRADING)
+        task_api.API(self.context).upgrade(self.id,
+                                           datastore_version.id)
+
 
 def create_server_list_matcher(server_list):
     # Returns a method which finds a server from the given list.
diff --git a/trove/instance/service.py b/trove/instance/service.py
index f3fb170a..4fcf77f5 100644
--- a/trove/instance/service.py
+++ b/trove/instance/service.py
@@ -312,15 +312,6 @@ class InstanceController(wsgi.Controller):
                 return configuration_id
 
     def _modify_instance(self, context, req, instance, **kwargs):
-        """Modifies the instance using the specified keyword arguments
-        'detach_replica': ignored if not present or False, if True,
-        specifies the instance is a replica that will be detached from
-        its master
-        'configuration_id': Ignored if not present, if None, detaches an
-        an attached configuration group, if not None, attaches the
-        specified configuration group
-        """
-
         if 'detach_replica' in kwargs and kwargs['detach_replica']:
             LOG.debug("Detaching replica from source.")
             context.notification = notification.DBaaSInstanceDetach(
@@ -342,6 +333,14 @@ class InstanceController(wsgi.Controller):
                                                                   request=req))
                 with StartNotification(context, instance_id=instance.id):
                     instance.unassign_configuration()
+        if 'datastore_version' in kwargs:
+            datastore_version = datastore_models.DatastoreVersion.load(
+                instance.datastore, kwargs['datastore_version'])
+            context.notification = (
+                notification.DBaaSInstanceUpgrade(context, request=req))
+            with StartNotification(context, instance_id=instance.id,
+                                   datastore_version_id=datastore_version.id):
+                instance.upgrade(datastore_version)
         if kwargs:
             instance.update_db(**kwargs)
 
@@ -381,6 +380,10 @@ class InstanceController(wsgi.Controller):
             args['name'] = body['instance']['name']
         if 'configuration' in body['instance']:
             args['configuration_id'] = self._configuration_parse(context, body)
+        if 'datastore_version' in body['instance']:
+            args['datastore_version'] = body['instance'].get(
+                'datastore_version')
+
         self._modify_instance(context, req, instance, **args)
         return wsgi.Result(None, 202)
 
diff --git a/trove/instance/tasks.py b/trove/instance/tasks.py
index 9bcb3290..8e6fdcd3 100644
--- a/trove/instance/tasks.py
+++ b/trove/instance/tasks.py
@@ -114,6 +114,7 @@ class InstanceTasks(object):
     SHRINKING_ERROR = InstanceTask(0x58, 'SHRINKING',
                                          'Shrinking Cluster Error.',
                                          is_error=True)
+    UPGRADING = InstanceTask(0x59, 'UPGRADING', 'Upgrading the instance.')
 
 # Dissuade further additions at run-time.
 InstanceTask.__init__ = None
diff --git a/trove/taskmanager/api.py b/trove/taskmanager/api.py
index 6fc1a29d..881574a2 100644
--- a/trove/taskmanager/api.py
+++ b/trove/taskmanager/api.py
@@ -198,6 +198,14 @@ class API(object):
 
         self._cast("delete_cluster", self.version_cap, cluster_id=cluster_id)
 
+    def upgrade(self, instance_id, datastore_version_id):
+        LOG.debug("Making async call to upgrade guest to datastore "
+                  "version %s " % datastore_version_id)
+
+        cctxt = self.client.prepare(version=self.version_cap)
+        cctxt.cast(self.context, "upgrade", instance_id=instance_id,
+                   datastore_version_id=datastore_version_id)
+
 
 def load(context, manager=None):
     if manager:
diff --git a/trove/taskmanager/manager.py b/trove/taskmanager/manager.py
index f6f79282..d375b19a 100644
--- a/trove/taskmanager/manager.py
+++ b/trove/taskmanager/manager.py
@@ -30,6 +30,7 @@ from trove.common import remote
 import trove.common.rpc.version as rpc_version
 from trove.common import server_group as srv_grp
 from trove.common.strategies.cluster import strategy
+from trove.datastore.models import DatastoreVersion
 import trove.extensions.mgmt.instances.models as mgmtmodels
 from trove.instance.tasks import InstanceTasks
 from trove.taskmanager import models
@@ -383,6 +384,12 @@ class Manager(periodic_task.PeriodicTasks):
                                   cluster_config, volume_type, modules,
                                   locality)
 
+    def upgrade(self, context, instance_id, datastore_version_id):
+        instance_tasks = models.BuiltInstanceTasks.load(context, instance_id)
+        datastore_version = DatastoreVersion.load_by_uuid(datastore_version_id)
+        with EndNotification(context):
+            instance_tasks.upgrade(datastore_version)
+
     def update_overrides(self, context, instance_id, overrides):
         instance_tasks = models.BuiltInstanceTasks.load(context, instance_id)
         instance_tasks.update_overrides(overrides)
diff --git a/trove/taskmanager/models.py b/trove/taskmanager/models.py
index ce6af071..1dae4d05 100644..100755
--- a/trove/taskmanager/models.py
+++ b/trove/taskmanager/models.py
@@ -336,32 +336,6 @@ class FreshInstanceTasks(FreshInstance, NotifyMixin, ConfigurationMixin):
 
         LOG.debug("End _delete_resource for instance %s" % self.id)
 
-    def _get_injected_files(self, datastore_manager):
-        injected_config_location = CONF.get('injected_config_location')
-        guest_info = CONF.get('guest_info')
-
-        if ('/' in guest_info):
-            # Set guest_info_file to exactly guest_info from the conf file.
-            # This should be /etc/guest_info for pre-Kilo compatibility.
-            guest_info_file = guest_info
-        else:
-            guest_info_file = os.path.join(injected_config_location,
-                                           guest_info)
-
-        files = {guest_info_file: (
-            "[DEFAULT]\n"
-            "guest_id=%s\n"
-            "datastore_manager=%s\n"
-            "tenant_id=%s\n"
-            % (self.id, datastore_manager, self.tenant_id))}
-
-        if os.path.isfile(CONF.get('guest_config')):
-            with open(CONF.get('guest_config'), "r") as f:
-                files[os.path.join(injected_config_location,
-                                   "trove-guestagent.conf")] = f.read()
-
-        return files
-
     def wait_for_instance(self, timeout, flavor):
         # Make sure the service becomes active before sending a usage
         # record to avoid over billing a customer for an instance that
@@ -421,7 +395,7 @@ class FreshInstanceTasks(FreshInstance, NotifyMixin, ConfigurationMixin):
                 LOG.debug("Successfully created security group for "
                           "instance: %s" % self.id)
 
-        files = self._get_injected_files(datastore_manager)
+        files = self.get_injected_files(datastore_manager)
         cinder_volume_type = volume_type or CONF.cinder_volume_type
         if use_heat:
             volume_info = self._create_server_volume_heat(
@@ -1420,6 +1394,58 @@ class BuiltInstanceTasks(BuiltInstance, NotifyMixin, ConfigurationMixin):
         datastore_status.status = rd_instance.ServiceStatuses.PAUSED
         datastore_status.save()
 
+    def upgrade(self, datastore_version):
+        LOG.debug("Upgrading instance %s to new datastore version %s",
+                  self, datastore_version)
+
+        def server_finished_rebuilding():
+            self.refresh_compute_server_info()
+            return not self.server_status_matches(['REBUILD'])
+
+        try:
+            upgrade_info = self.guest.pre_upgrade()
+
+            if self.volume_id:
+                volume = self.volume_client.volumes.get(self.volume_id)
+                volume_device = self._fix_device_path(
+                    volume.attachments[0]['device'])
+
+            injected_files = self.get_injected_files(
+                datastore_version.manager)
+            LOG.debug("Rebuilding instance %(instance)s with image %(image)s.",
+                      {'instance': self, 'image': datastore_version.image_id})
+            self.server.rebuild(datastore_version.image_id,
+                                files=injected_files)
+            utils.poll_until(
+                server_finished_rebuilding,
+                sleep_time=2, time_out=600)
+            if not self.server_status_matches(['ACTIVE']):
+                raise TroveError(_("Instance %(instance)s failed to "
+                                   "upgrade to %(datastore_version)s")
+                                 % {'instance': self,
+                                    'datastore_version': datastore_version})
+
+            if volume:
+                upgrade_info['device'] = volume_device
+
+            self.guest.post_upgrade(upgrade_info)
+
+            self.reset_task_status()
+
+        except Exception as e:
+            LOG.exception(e)
+            err = inst_models.InstanceTasks.BUILDING_ERROR_SERVER
+            self.update_db(task_status=err)
+            raise e
+
+    # Some cinder drivers appear to return "vdb" instead of "/dev/vdb".
+    # We need to account for that.
+    def _fix_device_path(self, device):
+        if device.startswith("/dev"):
+            return device
+        else:
+            return "/dev/%s" % device
+
 
 class BackupTasks(object):
     @classmethod
diff --git a/trove/tests/fakes/nova.py b/trove/tests/fakes/nova.py
index 4347d792..c4064315 100644
--- a/trove/tests/fakes/nova.py
+++ b/trove/tests/fakes/nova.py
@@ -17,6 +17,7 @@ from novaclient import exceptions as nova_exceptions
 from oslo_log import log as logging
 
 from trove.common.exception import PollTimeOut
+from trove.common.i18n import _
 from trove.common import instance as rd_instance
 from trove.tests.fakes.common import authorize
 
diff --git a/trove/tests/int_tests.py b/trove/tests/int_tests.py
index 94a08558..751904c2 100644
--- a/trove/tests/int_tests.py
+++ b/trove/tests/int_tests.py
@@ -42,6 +42,7 @@ from trove.tests.scenario.groups import instance_actions_group
 from trove.tests.scenario.groups import instance_create_group
 from trove.tests.scenario.groups import instance_delete_group
 from trove.tests.scenario.groups import instance_error_create_group
+from trove.tests.scenario.groups import instance_upgrade_group
 from trove.tests.scenario.groups import module_group
 from trove.tests.scenario.groups import negative_cluster_actions_group
 from trove.tests.scenario.groups import replication_group
@@ -146,6 +147,9 @@ instance_create_groups.extend([instance_create_group.GROUP,
 instance_error_create_groups = list(base_groups)
 instance_error_create_groups.extend([instance_error_create_group.GROUP])
 
+instance_upgrade_groups = list(instance_create_groups)
+instance_upgrade_groups.extend([instance_upgrade_group.GROUP])
+
 backup_groups = list(instance_create_groups)
 backup_groups.extend([groups.BACKUP,
                       groups.BACKUP_INST])
@@ -204,6 +208,7 @@ register(["guest_log"], guest_log_groups)
 register(["instance", "instance_actions"], instance_actions_groups)
 register(["instance_create"], instance_create_groups)
 register(["instance_error_create"], instance_error_create_groups)
+register(["instance_upgrade"], instance_upgrade_groups)
 register(["module"], module_groups)
 register(["module_create"], module_create_groups)
 register(["replication"], replication_groups)
@@ -228,8 +233,8 @@ register(["postgresql_supported"], common_groups,
          backup_incremental_groups, replication_groups)
 register(["mysql_supported", "percona_supported"], common_groups,
          backup_groups, configuration_groups, database_actions_groups,
-         replication_promote_groups, root_actions_groups, user_actions_groups,
-         backup_incremental_groups)
+         replication_promote_groups, instance_upgrade_groups,
+         root_actions_groups, user_actions_groups, backup_incremental_groups)
 register(["mariadb_supported"], common_groups,
          backup_groups, cluster_actions_groups, configuration_groups,
          database_actions_groups, replication_promote_groups,
diff --git a/trove/tests/scenario/groups/__init__.py b/trove/tests/scenario/groups/__init__.py
index 7a591ff8..48d4c41f 100644
--- a/trove/tests/scenario/groups/__init__.py
+++ b/trove/tests/scenario/groups/__init__.py
@@ -64,6 +64,10 @@ INST_ACTIONS_RESIZE = "scenario.inst_actions_resize_grp"
 INST_ACTIONS_RESIZE_WAIT = "scenario.inst_actions_resize_wait_grp"
 
 
+# Instance Upgrade Group
+INST_UPGRADE = "scenario.inst_upgrade_grp"
+
+
 # Instance Create Group
 INST_CREATE = "scenario.inst_create_grp"
 INST_CREATE_WAIT = "scenario.inst_create_wait_grp"
diff --git a/trove/tests/scenario/groups/configuration_group.py b/trove/tests/scenario/groups/configuration_group.py
index 3ddb5786..82894538 100644
--- a/trove/tests/scenario/groups/configuration_group.py
+++ b/trove/tests/scenario/groups/configuration_group.py
@@ -235,9 +235,11 @@ class ConfigurationInstCreateGroup(TestGroup):
       groups=[GROUP, groups.CFGGRP_INST,
               groups.CFGGRP_INST_CREATE_WAIT],
       runs_after_groups=[groups.INST_ACTIONS,
+                         groups.INST_UPGRADE,
                          groups.MODULE_INST_CREATE_WAIT])
 class ConfigurationInstCreateWaitGroup(TestGroup):
     """Test that Instance Configuration Group Create Completes."""
+
     def __init__(self):
         super(ConfigurationInstCreateWaitGroup, self).__init__(
             ConfigurationRunnerFactory.instance())
diff --git a/trove/tests/scenario/groups/instance_actions_group.py b/trove/tests/scenario/groups/instance_actions_group.py
index f7bc21d9..3730c24a 100644
--- a/trove/tests/scenario/groups/instance_actions_group.py
+++ b/trove/tests/scenario/groups/instance_actions_group.py
@@ -54,6 +54,7 @@ class InstanceActionsGroup(TestGroup):
 @test(depends_on_groups=[groups.INST_CREATE_WAIT],
       groups=[GROUP, groups.INST_ACTIONS_RESIZE],
       runs_after_groups=[groups.INST_ACTIONS,
+                         groups.INST_UPGRADE,
                          groups.MODULE_INST_CREATE_WAIT,
                          groups.CFGGRP_INST_CREATE_WAIT,
                          groups.BACKUP_CREATE,
diff --git a/trove/tests/scenario/groups/instance_delete_group.py b/trove/tests/scenario/groups/instance_delete_group.py
index d29a23ab..40af6e31 100644
--- a/trove/tests/scenario/groups/instance_delete_group.py
+++ b/trove/tests/scenario/groups/instance_delete_group.py
@@ -33,6 +33,7 @@ class InstanceDeleteRunnerFactory(test_runners.RunnerFactory):
       groups=[GROUP, groups.INST_DELETE],
       runs_after_groups=[groups.INST_INIT_DELETE,
                          groups.INST_ACTIONS,
+                         groups.INST_UPGRADE,
                          groups.INST_ACTIONS_RESIZE_WAIT,
                          groups.BACKUP_INST_DELETE,
                          groups.BACKUP_INC_INST_DELETE,
diff --git a/trove/tests/scenario/groups/instance_upgrade_group.py b/trove/tests/scenario/groups/instance_upgrade_group.py
new file mode 100644
index 00000000..c0d00ba4
--- /dev/null
+++ b/trove/tests/scenario/groups/instance_upgrade_group.py
@@ -0,0 +1,92 @@
+# Copyright 2015 Tesora Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from proboscis import test
+
+from trove.tests.scenario import groups
+from trove.tests.scenario.groups.test_group import TestGroup
+from trove.tests.scenario.runners import test_runners
+
+
+GROUP = "scenario.instance_upgrade_group"
+
+
+class InstanceUpgradeRunnerFactory(test_runners.RunnerFactory):
+
+    _runner_ns = 'instance_upgrade_runners'
+    _runner_cls = 'InstanceUpgradeRunner'
+
+
+class UserActionsRunnerFactory(test_runners.RunnerFactory):
+
+    _runner_ns = 'user_actions_runners'
+    _runner_cls = 'UserActionsRunner'
+
+
+class DatabaseActionsRunnerFactory(test_runners.RunnerFactory):
+
+    _runner_ns = 'database_actions_runners'
+    _runner_cls = 'DatabaseActionsRunner'
+
+
+@test(depends_on_groups=[groups.INST_CREATE_WAIT],
+      groups=[GROUP, groups.INST_UPGRADE],
+      runs_after_groups=[groups.INST_ACTIONS])
+class InstanceUpgradeGroup(TestGroup):
+
+    def __init__(self):
+        super(InstanceUpgradeGroup, self).__init__(
+            InstanceUpgradeRunnerFactory.instance())
+        self.database_actions_runner = DatabaseActionsRunnerFactory.instance()
+        self.user_actions_runner = UserActionsRunnerFactory.instance()
+
+    @test
+    def create_user_databases(self):
+        """Create user databases on an existing instance."""
+        # These databases may be referenced by the users (below) so we need to
+        # create them first.
+        self.database_actions_runner.run_databases_create()
+
+    @test(runs_after=[create_user_databases])
+    def create_users(self):
+        """Create users on an existing instance."""
+        self.user_actions_runner.run_users_create()
+
+    @test(runs_after=[create_users])
+    def instance_upgrade(self):
+        """Upgrade an existing instance."""
+        self.test_runner.run_instance_upgrade()
+
+    @test(depends_on=[instance_upgrade])
+    def show_user(self):
+        """Show created users."""
+        self.user_actions_runner.run_user_show()
+
+    @test(depends_on=[create_users],
+          runs_after=[show_user])
+    def list_users(self):
+        """List the created users."""
+        self.user_actions_runner.run_users_list()
+
+    @test(depends_on=[create_users],
+          runs_after=[list_users])
+    def delete_user(self):
+        """Delete the created users."""
+        self.user_actions_runner.run_user_delete()
+
+    @test(depends_on=[create_user_databases], runs_after=[delete_user])
+    def delete_user_databases(self):
+        """Delete the user databases."""
+        self.database_actions_runner.run_database_delete()
diff --git a/trove/tests/scenario/groups/module_group.py b/trove/tests/scenario/groups/module_group.py
index 4e58c380..495d12a9 100644
--- a/trove/tests/scenario/groups/module_group.py
+++ b/trove/tests/scenario/groups/module_group.py
@@ -374,7 +374,7 @@ class ModuleInstCreateGroup(TestGroup):
 
 @test(depends_on_groups=[groups.MODULE_INST_CREATE],
       groups=[GROUP, groups.MODULE_INST, groups.MODULE_INST_CREATE_WAIT],
-      runs_after_groups=[groups.INST_ACTIONS])
+      runs_after_groups=[groups.INST_ACTIONS, groups.INST_UPGRADE])
 class ModuleInstCreateWaitGroup(TestGroup):
     """Test that Module Instance Create Completes."""
 
diff --git a/trove/tests/scenario/runners/instance_upgrade_runners.py b/trove/tests/scenario/runners/instance_upgrade_runners.py
new file mode 100644
index 00000000..587024de
--- /dev/null
+++ b/trove/tests/scenario/runners/instance_upgrade_runners.py
@@ -0,0 +1,33 @@
+# Copyright 2015 Tesora Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from trove.tests.scenario.runners.test_runners import TestRunner
+
+
+class InstanceUpgradeRunner(TestRunner):
+
+    def __init__(self):
+        super(InstanceUpgradeRunner, self).__init__()
+
+    def run_instance_upgrade(
+            self, expected_states=['UPGRADE', 'ACTIVE'],
+            expected_http_code=202):
+        instance_id = self.instance_info.id
+        self.report.log("Testing upgrade on instance: %s" % instance_id)
+
+        target_version = self.instance_info.dbaas_datastore_version
+        self.auth_client.instances.upgrade(instance_id, target_version)
+        self.assert_instance_action(instance_id, expected_states,
+                                    expected_http_code)
diff --git a/trove/tests/unittests/guestagent/test_dbaas.py b/trove/tests/unittests/guestagent/test_dbaas.py
index 588b3c77..15ead69d 100644
--- a/trove/tests/unittests/guestagent/test_dbaas.py
+++ b/trove/tests/unittests/guestagent/test_dbaas.py
@@ -1548,10 +1548,12 @@ class MySqlAppMockTest(trove_testtools.TestCase):
         utils.execute_with_timeout = self.orig_utils_execute_with_timeout
         super(MySqlAppMockTest, self).tearDown()
 
+    @patch('trove.guestagent.common.configuration.ConfigurationManager'
+           '.refresh_cache')
     @patch.object(mysql_common_service, 'clear_expired_password')
     @patch.object(utils, 'generate_random_password',
                   return_value='some_password')
-    def test_secure_keep_root(self, auth_pwd_mock, clear_pwd_mock):
+    def test_secure_keep_root(self, auth_pwd_mock, clear_pwd_mock, _):
         with patch.object(self.mock_client,
                           'execute', return_value=None) as mock_execute:
             utils.execute_with_timeout = MagicMock(return_value=None)
@@ -1569,10 +1571,12 @@ class MySqlAppMockTest(trove_testtools.TestCase):
                 app._reset_configuration.assert_has_calls(reset_config_calls)
                 self.assertTrue(mock_execute.called)
 
+    @patch('trove.guestagent.common.configuration.ConfigurationManager'
+           '.refresh_cache')
     @patch.object(mysql_common_service, 'clear_expired_password')
     @patch.object(mysql_common_service.BaseMySqlApp,
                   'get_auth_password', return_value='some_password')
-    def test_secure_with_mycnf_error(self, auth_pwd_mock, clear_pwd_mock):
+    def test_secure_with_mycnf_error(self, *args):
         with patch.object(self.mock_client,
                           'execute', return_value=None) as mock_execute:
             with patch.object(operating_system, 'service_discovery',
diff --git a/trove/tests/unittests/instance/test_instance_models.py b/trove/tests/unittests/instance/test_instance_models.py
index f39b9e8d..30a7c107 100644
--- a/trove/tests/unittests/instance/test_instance_models.py
+++ b/trove/tests/unittests/instance/test_instance_models.py
@@ -250,6 +250,78 @@ class CreateInstanceTest(trove_testtools.TestCase):
         self.assertIsNotNone(instance)
 
 
+class TestInstanceUpgrade(trove_testtools.TestCase):
+
+    def setUp(self):
+        self.context = trove_testtools.TroveTestContext(self, is_admin=True)
+        util.init_db()
+
+        self.datastore = datastore_models.DBDatastore.create(
+            id=str(uuid.uuid4()),
+            name='test' + str(uuid.uuid4()),
+            default_version_id=str(uuid.uuid4()))
+
+        self.datastore_version1 = datastore_models.DBDatastoreVersion.create(
+            id=self.datastore.default_version_id,
+            name='name' + str(uuid.uuid4()),
+            image_id='old_image',
+            packages=str(uuid.uuid4()),
+            datastore_id=self.datastore.id,
+            manager='test',
+            active=1)
+
+        self.datastore_version2 = datastore_models.DBDatastoreVersion.create(
+            id=str(uuid.uuid4()),
+            name='name' + str(uuid.uuid4()),
+            image_id='new_image',
+            packages=str(uuid.uuid4()),
+            datastore_id=self.datastore.id,
+            manager='test',
+            active=1)
+
+        self.safe_nova_client = models.create_nova_client
+        models.create_nova_client = nova.fake_create_nova_client
+        super(TestInstanceUpgrade, self).setUp()
+
+    def tearDown(self):
+        self.datastore.delete()
+        self.datastore_version1.delete()
+        self.datastore_version2.delete()
+        models.create_nova_client = self.safe_nova_client
+        super(TestInstanceUpgrade, self).tearDown()
+
+    @patch.object(task_api.API, 'get_client', Mock(return_value=Mock()))
+    @patch.object(task_api.API, 'upgrade')
+    def test_upgrade(self, task_upgrade):
+        instance_model = DBInstance(
+            InstanceTasks.NONE,
+            id=str(uuid.uuid4()),
+            name="TestUpgradeInstance",
+            datastore_version_id=self.datastore_version1.id)
+        instance_model.set_task_status(InstanceTasks.NONE)
+        instance_model.save()
+        instance_status = InstanceServiceStatus(
+            ServiceStatuses.RUNNING,
+            id=str(uuid.uuid4()),
+            instance_id=instance_model.id)
+        instance_status.save()
+        self.assertIsNotNone(instance_model)
+        instance = models.load_instance(models.Instance, self.context,
+                                        instance_model.id)
+
+        try:
+            instance.upgrade(self.datastore_version2)
+
+            self.assertEqual(self.datastore_version2.id,
+                             instance.db_info.datastore_version_id)
+            self.assertEqual(InstanceTasks.UPGRADING,
+                             instance.db_info.task_status)
+            self.assertTrue(task_upgrade.called)
+        finally:
+            instance_status.delete()
+            instance_model.delete()
+
+
 class TestReplication(trove_testtools.TestCase):
 
     def setUp(self):
diff --git a/trove/tests/unittests/taskmanager/test_api.py b/trove/tests/unittests/taskmanager/test_api.py
index ab11e510..48103fdc 100644
--- a/trove/tests/unittests/taskmanager/test_api.py
+++ b/trove/tests/unittests/taskmanager/test_api.py
@@ -122,6 +122,14 @@ class ApiTest(trove_testtools.TestCase):
                                 ('Could not transform %s' % flavor),
                                 self.api._transform_obj, flavor)
 
+    def test_upgrade(self):
+        self.api.upgrade('some-instance-id', 'some-datastore-version')
+
+        self._verify_rpc_prepare_before_cast()
+        self._verify_cast('upgrade',
+                          instance_id='some-instance-id',
+                          datastore_version_id='some-datastore-version')
+
 
 class TestAPI(trove_testtools.TestCase):
 
diff --git a/trove/tests/unittests/taskmanager/test_models.py b/trove/tests/unittests/taskmanager/test_models.py
index 359b3582..36315d3a 100644
--- a/trove/tests/unittests/taskmanager/test_models.py
+++ b/trove/tests/unittests/taskmanager/test_models.py
@@ -18,6 +18,7 @@ import uuid
 
 from cinderclient import exceptions as cinder_exceptions
 import cinderclient.v2.client as cinderclient
+from cinderclient.v2 import volumes as cinderclient_volumes
 from mock import Mock, MagicMock, patch, PropertyMock, call
 from novaclient import exceptions as nova_exceptions
 import novaclient.v2.flavors
@@ -217,6 +218,9 @@ class FreshInstanceTasksTest(trove_testtools.TestCase):
         self.task_models_conf_patch = patch('trove.taskmanager.models.CONF')
         self.task_models_conf_mock = self.task_models_conf_patch.start()
         self.addCleanup(self.task_models_conf_patch.stop)
+        self.inst_models_conf_patch = patch('trove.instance.models.CONF')
+        self.inst_models_conf_mock = self.inst_models_conf_patch.start()
+        self.addCleanup(self.inst_models_conf_patch.stop)
 
     def tearDown(self):
         super(FreshInstanceTasksTest, self).tearDown()
@@ -252,9 +256,9 @@ class FreshInstanceTasksTest(trove_testtools.TestCase):
             else:
                 return ''
 
-        self.task_models_conf_mock.get.side_effect = fake_conf_getter
+        self.inst_models_conf_mock.get.side_effect = fake_conf_getter
         # execute
-        files = self.freshinstancetasks._get_injected_files("test")
+        files = self.freshinstancetasks.get_injected_files("test")
         # verify
         self.assertTrue(
             '/etc/trove/conf.d/guest_info.conf' in files)
@@ -275,9 +279,9 @@ class FreshInstanceTasksTest(trove_testtools.TestCase):
             else:
                 return ''
 
-        self.task_models_conf_mock.get.side_effect = fake_conf_getter
+        self.inst_models_conf_mock.get.side_effect = fake_conf_getter
         # execute
-        files = self.freshinstancetasks._get_injected_files("test")
+        files = self.freshinstancetasks.get_injected_files("test")
         # verify
         self.assertTrue(
             '/etc/guest_info' in files)
@@ -396,7 +400,7 @@ class FreshInstanceTasksTest(trove_testtools.TestCase):
     @patch.object(BaseInstance, 'update_db')
     @patch.object(backup_models.Backup, 'get_by_id')
     @patch.object(taskmanager_models.FreshInstanceTasks, 'report_root_enabled')
-    @patch.object(taskmanager_models.FreshInstanceTasks, '_get_injected_files')
+    @patch.object(taskmanager_models.FreshInstanceTasks, 'get_injected_files')
     @patch.object(taskmanager_models.FreshInstanceTasks, '_create_secgroup')
     @patch.object(taskmanager_models.FreshInstanceTasks, '_build_volume_info')
     @patch.object(taskmanager_models.FreshInstanceTasks, '_create_server')
@@ -417,7 +421,7 @@ class FreshInstanceTasksTest(trove_testtools.TestCase):
 
     @patch.object(BaseInstance, 'update_db')
     @patch.object(taskmanager_models.FreshInstanceTasks, '_create_dns_entry')
-    @patch.object(taskmanager_models.FreshInstanceTasks, '_get_injected_files')
+    @patch.object(taskmanager_models.FreshInstanceTasks, 'get_injected_files')
     @patch.object(taskmanager_models.FreshInstanceTasks, '_create_server')
     @patch.object(taskmanager_models.FreshInstanceTasks, '_create_secgroup')
     @patch.object(taskmanager_models.FreshInstanceTasks, '_build_volume_info')
@@ -691,6 +695,10 @@ class BuiltInstanceTasksTest(trove_testtools.TestCase):
                                                    True)
         stub_flavor_manager.get = MagicMock(return_value=nova_flavor)
 
+        self.instance_task._volume_client = MagicMock(spec=cinderclient)
+        self.instance_task._volume_client.volumes = Mock(
+            spec=cinderclient_volumes.VolumeManager)
+
         answers = (status for status in
                    self.get_inst_service_status('inst_stat-id',
                                                 [ServiceStatuses.SHUTDOWN,
@@ -917,6 +925,36 @@ class BuiltInstanceTasksTest(trove_testtools.TestCase):
         self.instance_task.demote_replication_master()
         self.instance_task._guest.demote_replication_master.assert_any_call()
 
+    @patch.multiple(taskmanager_models.BuiltInstanceTasks,
+                    get_injected_files=Mock(return_value="the-files"))
+    def test_upgrade(self, *args):
+        pre_rebuild_server = self.instance_task.server
+        dsv = Mock(image_id='foo_image')
+        mock_volume = Mock(attachments=[{'device': '/dev/mock_dev'}])
+        with patch.object(self.instance_task._volume_client.volumes, "get",
+                          Mock(return_value=mock_volume)):
+            mock_server = Mock(status='ACTIVE')
+            with patch.object(self.instance_task._nova_client.servers,
+                              'get', Mock(return_value=mock_server)):
+                with patch.multiple(self.instance_task._guest,
+                                    pre_upgrade=Mock(return_value={}),
+                                    post_upgrade=Mock()):
+                    self.instance_task.upgrade(dsv)
+
+                    self.instance_task._guest.pre_upgrade.assert_called_with()
+                    pre_rebuild_server.rebuild.assert_called_with(
+                        dsv.image_id, files="the-files")
+                    self.instance_task._guest.post_upgrade.assert_called_with(
+                        mock_volume.attachments[0])
+
+    def test_fix_device_path(self):
+        self.assertEqual("/dev/vdb", self.instance_task.
+                         _fix_device_path("vdb"))
+        self.assertEqual("/dev/dev", self.instance_task.
+                         _fix_device_path("dev"))
+        self.assertEqual("/dev/vdb/dev", self.instance_task.
+                         _fix_device_path("vdb/dev"))
+
 
 class BackupTasksTest(trove_testtools.TestCase):