Keep user defined configuration after resizing instance

* Never remove user defined config, changing the function name from
  save_configuration to reset_configuration in trove-guestagent.
* Improved some logs
* Do not remove Innodb Log Files after resize which will cause error:
  Can't open and lock privilege tables: Table './mysql/user' is marked
  as crashed and should be repaired

Story: 2009033
Task: 42773
Change-Id: I9e3165ed9b38b15714542e35456415e65d438497
(cherry picked from commit c274ab9f1a9b270c0ec2cb2d4aebe052aa37d416)

9 files changed, 53 insertions, 47 deletions

diff --git a/trove/guestagent/api.py b/trove/guestagent/api.py
index eb50b464..fc146100 100644
--- a/trove/guestagent/api.py
+++ b/trove/guestagent/api.py
@@ -408,7 +408,10 @@ class API(object):
         self._call("restart", self.agent_high_timeout, version=version)
 
     def start_db_with_conf_changes(self, config_contents, ds_version):
-        """Start the database server."""
+        """Start the database with given configuration.
+
+        This method is called after resize.
+        """
         LOG.debug("Sending the call to start the database process on "
                   "the Guest with a timeout of %s.",
                   self.agent_high_timeout)
@@ -424,8 +427,10 @@ class API(object):
                    version=version, **start_args)
 
     def reset_configuration(self, configuration):
-        """Ignore running state of the database server; just change
-           the config file to a new flavor.
+        """Reset the database base configuration.
+
+        Ignore running state of the database server, just change the config
+        file to a new flavor.
         """
         LOG.debug("Sending the call to change the database conf file on the "
                   "Guest with a timeout of %s.",
diff --git a/trove/guestagent/common/configuration.py b/trove/guestagent/common/configuration.py
index 58595eb1..f26f0ef2 100644
--- a/trove/guestagent/common/configuration.py
+++ b/trove/guestagent/common/configuration.py
@@ -133,20 +133,23 @@ class ConfigurationManager(object):
 
         return base_options
 
-    def save_configuration(self, options):
+    def reset_configuration(self, options, remove_overrides=False):
         """Write given contents to the base configuration file.
-        Remove all existing overrides (both system and user).
 
-        :param options        Contents of the configuration file.
-        :type options         string or dict
+        Remove all existing overrides (both system and user) as required.
+
+        :param options: Contents of the configuration file (string or dict).
+        :param remove_overrides: Remove the overrides or not.
         """
         if isinstance(options, dict):
             # Serialize a dict of options for writing.
-            self.save_configuration(self._codec.serialize(options))
+            self.reset_configuration(self._codec.serialize(options),
+                                     remove_overrides=remove_overrides)
         else:
-            self._override_strategy.remove(self.USER_GROUP)
-            self._override_strategy.remove(self.SYSTEM_PRE_USER_GROUP)
-            self._override_strategy.remove(self.SYSTEM_POST_USER_GROUP)
+            if remove_overrides:
+                self._override_strategy.remove(self.USER_GROUP)
+                self._override_strategy.remove(self.SYSTEM_PRE_USER_GROUP)
+                self._override_strategy.remove(self.SYSTEM_POST_USER_GROUP)
 
             operating_system.write_file(
                 self._base_config_path, options, as_root=self._requires_root)
diff --git a/trove/guestagent/datastore/manager.py b/trove/guestagent/datastore/manager.py
index 65281510..a722159a 100644
--- a/trove/guestagent/datastore/manager.py
+++ b/trove/guestagent/datastore/manager.py
@@ -317,6 +317,10 @@ class Manager(periodic_task.PeriodicTasks):
         pass
 
     def start_db_with_conf_changes(self, context, config_contents, ds_version):
+        """Start the database with given configuration.
+
+        This method is called after resize.
+        """
         self.app.start_db_with_conf_changes(config_contents, ds_version)
 
     def stop_db(self, context):
@@ -381,15 +385,17 @@ class Manager(periodic_task.PeriodicTasks):
     # Configuration
     ###############
     def reset_configuration(self, context, configuration):
-        """The default implementation should be sufficient if a
+        """Reset database base configuration.
+
+        The default implementation should be sufficient if a
         configuration_manager is provided. Even if one is not, this
         method needs to be implemented to allow the rollback of
         flavor-resize on the guestagent side.
         """
-        LOG.info("Resetting configuration.")
         if self.configuration_manager:
+            LOG.info("Resetting configuration.")
             config_contents = configuration['config_contents']
-            self.configuration_manager.save_configuration(config_contents)
+            self.configuration_manager.reset_configuration(config_contents)
 
     def apply_overrides_on_prepare(self, context, overrides):
         self.update_overrides(context, overrides)
diff --git a/trove/guestagent/datastore/mysql_common/manager.py b/trove/guestagent/datastore/mysql_common/manager.py
index 31172f20..5c55ebe6 100644
--- a/trove/guestagent/datastore/mysql_common/manager.py
+++ b/trove/guestagent/datastore/mysql_common/manager.py
@@ -76,7 +76,7 @@ class MySqlManager(manager.Manager):
 
         # Prepare mysql configuration
         LOG.info('Preparing database configuration')
-        self.app.configuration_manager.save_configuration(config_contents)
+        self.app.configuration_manager.reset_configuration(config_contents)
         self.app.update_overrides(overrides)
 
         # Restore data from backup and reset root password
@@ -311,7 +311,7 @@ class MySqlManager(manager.Manager):
         try:
             # Prepare mysql configuration
             LOG.debug('Preparing database configuration')
-            self.app.configuration_manager.save_configuration(config_contents)
+            self.app.configuration_manager.reset_configuration(config_contents)
             self.app.update_overrides(config_overrides)
 
             # Start database service.
diff --git a/trove/guestagent/datastore/mysql_common/service.py b/trove/guestagent/datastore/mysql_common/service.py
index 339170bb..84a514c1 100644
--- a/trove/guestagent/datastore/mysql_common/service.py
+++ b/trove/guestagent/datastore/mysql_common/service.py
@@ -629,11 +629,6 @@ class BaseMySqlApp(service.BaseDbApp):
                 LOG.exception("Could not delete logfile.")
                 raise
 
-    def reset_configuration(self, configuration):
-        LOG.info("Resetting configuration.")
-        self.configuration_manager.save_configuration(configuration)
-        self.wipe_ib_logfiles()
-
     def restart(self):
         LOG.info("Restarting mysql")
 
diff --git a/trove/guestagent/datastore/postgres/manager.py b/trove/guestagent/datastore/postgres/manager.py
index 2a2da7f0..b233d5b3 100644
--- a/trove/guestagent/datastore/postgres/manager.py
+++ b/trove/guestagent/datastore/postgres/manager.py
@@ -54,7 +54,7 @@ class PostgresManager(manager.Manager):
                                           as_root=True)
 
         LOG.info('Preparing database config files')
-        self.app.configuration_manager.save_configuration(config_contents)
+        self.app.configuration_manager.reset_configuration(config_contents)
         self.app.set_data_dir(self.app.datadir)
         self.app.update_overrides(overrides)
 
diff --git a/trove/guestagent/datastore/postgres/service.py b/trove/guestagent/datastore/postgres/service.py
index 404db303..a652215b 100644
--- a/trove/guestagent/datastore/postgres/service.py
+++ b/trove/guestagent/datastore/postgres/service.py
@@ -147,9 +147,6 @@ class PgSqlApp(service.BaseDbApp):
         if overrides:
             self.configuration_manager.apply_user_override(overrides)
 
-    def reset_configuration(self, configuration):
-        self.configuration_manager.save_configuration(configuration)
-
     def apply_overrides(self, overrides):
         """Reload config."""
         cmd = "pg_ctl reload"
diff --git a/trove/guestagent/datastore/service.py b/trove/guestagent/datastore/service.py
index f1c3c3ca..d52d3cef 100644
--- a/trove/guestagent/datastore/service.py
+++ b/trove/guestagent/datastore/service.py
@@ -378,7 +378,8 @@ class BaseDbApp(object):
         self.configuration_manager.remove_user_override()
 
     def reset_configuration(self, configuration):
-        pass
+        LOG.info("Resetting configuration.")
+        self.configuration_manager.reset_configuration(configuration)
 
     def stop_db(self, update_db=False):
         LOG.info("Stopping database.")
@@ -396,6 +397,10 @@ class BaseDbApp(object):
             raise exception.TroveError("Failed to stop database")
 
     def start_db_with_conf_changes(self, config_contents, ds_version):
+        """Start the database with given configuration.
+
+        This method is called after resize.
+        """
         LOG.info(f"Starting database service with new configuration and "
                  f"datastore version {ds_version}.")
 
diff --git a/trove/taskmanager/models.py b/trove/taskmanager/models.py
index 386c7aa6..3d2fc52a 100755
--- a/trove/taskmanager/models.py
+++ b/trove/taskmanager/models.py
@@ -1190,7 +1190,7 @@ class BuiltInstanceTasks(BuiltInstance, NotifyMixin, ConfigurationMixin):
                 self.reset_task_status()
 
     def attach_replica(self, master, restart=False):
-        LOG.info("Attaching replica %s to master %s", self.id, master.id)
+        LOG.info("Attaching replica %s to primary %s", self.id, master.id)
         try:
             replica_info = master.guest.get_replica_context()
             flavor = self.nova_client.flavors.get(self.flavor_id)
@@ -1847,15 +1847,17 @@ class ResizeActionBase(object):
         pass
 
     def _assert_datastore_is_ok(self):
+        LOG.info(f"Re-config database for instance {self.instance.id} after "
+                 f"resize")
         self._start_datastore()
 
     def _assert_processes_are_ok(self):
         """Checks the procs; if anything is wrong, reverts the operation."""
         # Tell the guest to turn back on, and make sure it can start.
+        LOG.info(f"Waiting for database status changed after resizing "
+                 f"{self.instance.id}")
         self._assert_guest_is_ok()
-        LOG.debug("Nova guest is ok.")
         self._assert_datastore_is_ok()
-        LOG.debug("Datastore is ok.")
 
     def _confirm_nova_action(self):
         LOG.debug("Instance %s calling Compute confirm resize...",
@@ -1884,7 +1886,7 @@ class ResizeActionBase(object):
     def execute(self):
         """Initiates the action."""
         try:
-            LOG.debug("Instance %s calling stop_db...", self.instance.id)
+            LOG.info(f"Stopping database service for {self.instance.id}")
             self.instance.guest.stop_db(do_not_start_on_reboot=True)
         except Exception as e:
             if self.ignore_stop_error:
@@ -1918,25 +1920,17 @@ class ResizeActionBase(object):
 
     def _perform_nova_action(self):
         """Calls Nova to resize or migrate an instance, and confirms."""
-        LOG.debug("Begin resize method _perform_nova_action instance: %s",
-                  self.instance.id)
         need_to_revert = False
         try:
-            LOG.debug("Initiating nova action")
             self._initiate_nova_action()
-            LOG.debug("Waiting for nova action completed")
             self._wait_for_nova_action()
-            LOG.debug("Asserting nova status is ok")
             self._assert_nova_status_is_ok()
             need_to_revert = True
-            LOG.debug("Asserting nova action success")
             self._assert_nova_action_was_successful()
-            LOG.debug("Asserting processes are OK")
             self._assert_processes_are_ok()
-            LOG.debug("Confirming nova action")
             self._confirm_nova_action()
         except Exception:
-            LOG.exception("Exception during nova action.")
+            LOG.exception(f"Failed to resize instance {self.instance.id}")
             if need_to_revert:
                 LOG.error("Reverting action for instance %s",
                           self.instance.id)
@@ -1944,13 +1938,12 @@ class ResizeActionBase(object):
                 self._wait_for_revert_nova_action()
 
             if self.instance.server_status_matches(['ACTIVE']):
-                LOG.error("Restarting datastore.")
+                LOG.error(f"Restarting instance {self.instance.id}")
                 self.instance.guest.restart()
             else:
-                LOG.error("Cannot restart datastore because "
-                          "Nova server status is not ACTIVE")
+                LOG.error(f"Cannot restart instance {self.instance.id} "
+                          f"because Nova server status is not ACTIVE")
 
-            LOG.error("Error resizing instance %s.", self.instance.id)
             raise
 
         self._record_action_success()
@@ -1958,8 +1951,8 @@ class ResizeActionBase(object):
                   self.instance.id)
 
     def _wait_for_nova_action(self):
-        LOG.info(f"Waiting for Nova server status changed to "
-                 f"{self.wait_status}")
+        LOG.debug(f"Waiting for Nova server status changed to "
+                  f"{self.wait_status} for {self.instance.id}")
 
         def update_server_info():
             self.instance.refresh_compute_server_info()
@@ -2005,6 +1998,7 @@ class ResizeAction(ResizeActionBase):
             raise TroveError(msg)
 
     def _initiate_nova_action(self):
+        LOG.info(f"Resizing Nova server for instance {self.instance.id}")
         self.instance.server.resize(self.new_flavor_id)
 
     def _revert_nova_action(self):
@@ -2078,7 +2072,7 @@ class RebuildAction(ResizeActionBase):
             self.instance.datastore.name,
             self.instance.datastore_version.name)
 
-        LOG.debug(f"Rebuilding Nova server {self.instance.server.id}")
+        LOG.info(f"Rebuilding Nova server for instance {self.instance.id}")
         # Before Nova version 2.57, userdata is not supported when doing
         # rebuild, have to use injected files instead.
         self.instance.server.rebuild(
@@ -2111,7 +2105,8 @@ class RebuildAction(ResizeActionBase):
             self.instance.datastore_version.name,
             config_contents=config_contents, config_overrides=overrides)
 
-        LOG.info(f"Waiting for instance {self.instance.id} healthy")
+        LOG.info(f"Waiting for instance {self.instance.id} healthy after "
+                 f"rebuild")
         self._assert_guest_is_ok()
         self.wait_for_healthy()
         LOG.info(f"Finished to rebuild {self.instance.id}")