Fix for the following bugs:

  - BUG#22306: STOP INSTANCE can not be applied for instances in Crashed,
    Failed and Abandoned;
  - BUG#23476: DROP INSTANCE does not work
  - BUG#23215: STOP INSTANCE takes too much time

BUG#22306:
The problem was that STOP INSTANCE checked that mysqld is up and running.
If it was not so, STOP INSTANCE reported an error. Now, STOP INSTANCE
reports an error if the instance has been started (mysqld can be down).

BUG#23476:
The problem was that DROP INSTANCE tried to stop inactive instance. The fix is
trivial.

BUG#23215:
The problem was that locks were not acquired properly, so the
instance-monitoring thread could not acquire the mutex, holded by the
query-processing thread.

The fix is to simplify locking scheme by moving instance-related information to
Instance-class out of Guardian-class. This allows to get rid of storing a
separate list of Instance-information in Guardian and keeping it synchronized
with the original list in Instance_map.


server-tools/instance-manager/commands.cc:
  1. Introduce Instance_cmd class -- base class for the commands
     that deal with the one instance;
  2. Remove Instance_map argument from command constructors;
  3. Ensure, that Instance Map and Instance are locked in the proper order;
  4. Polishing.
server-tools/instance-manager/commands.h:
  1. Introduce Instance_cmd class -- base class for the commands
     that deal with the one instance;
  2. Remove Instance_map argument from command constructors;
  3. Polishing.
server-tools/instance-manager/guardian.cc:
  1. Move "extended" instance information to the Instance-class.
     That allows to get rid of storing instance-related container and data in
     Guardian class, that significantly simplifies locking schema.
  2. Polishing.
server-tools/instance-manager/guardian.h:
  1. Move "extended" instance information to the Instance-class.
     That allows to get rid of storing instance-related container and data in
     Guardian class, that significantly simplifies locking schema.
  2. Polishing.
server-tools/instance-manager/instance.cc:
  1. Move "extended" instance information to the Instance-class.
  2. Introduce new state STOPPED to mark that guarded instance
     is stopped and should not be restarted by Guardian.
  3. Polishing.
server-tools/instance-manager/instance.h:
  1. Move "extended" instance information to the Instance-class.
  2. Introduce new state STOPPED to mark that guarded instance
     is stopped and should not be restarted by Guardian.
  3. Polishing.
server-tools/instance-manager/instance_map.cc:
  1. Move flush_instances() from Instance_map to Manager.
  2. Polishing.
server-tools/instance-manager/instance_map.h:
  1. Move flush_instances() from Instance_map to Manager.
  2. Polishing.
server-tools/instance-manager/instance_options.h:
  Polishing.
server-tools/instance-manager/manager.cc:
  1. Move flush_instances() from Instance_map to Manager.
  2. Polishing.
server-tools/instance-manager/manager.h:
  1. Move flush_instances() from Instance_map to Manager.
  2. Polishing.
server-tools/instance-manager/user_map.cc:
  Polishing.

1 files changed, 51 insertions, 74 deletions

diff --git a/server-tools/instance-manager/guardian.h b/server-tools/instance-manager/guardian.h
index 0eee1dc631d..2c7987f4565 100644
--- a/server-tools/instance-manager/guardian.h
+++ b/server-tools/instance-manager/guardian.h
@@ -17,10 +17,12 @@
    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */
 
 
-#include "thread_registry.h"
+#include <my_global.h>
 #include <my_sys.h>
 #include <my_list.h>
 
+#include "thread_registry.h"
+
 #if defined(__GNUC__) && defined(USE_PRAGMA_INTERFACE)
 #pragma interface
 #endif
@@ -28,7 +30,6 @@
 class Instance;
 class Instance_map;
 class Thread_registry;
-struct GUARD_NODE;
 
 /**
   The guardian thread is responsible for monitoring and restarting of guarded
@@ -38,97 +39,73 @@ struct GUARD_NODE;
 class Guardian: public Thread
 {
 public:
-  /* states of an instance */
-  enum enum_instance_state { NOT_STARTED= 1, STARTING, STARTED, JUST_CRASHED,
-                             CRASHED, CRASHED_AND_ABANDONED, STOPPING };
-
-  /*
-    The Guardian list node structure. Guardian utilizes it to store
-    guarded instances plus some additional info.
-  */
+  Guardian(Thread_registry *thread_registry_arg,
+           Instance_map *instance_map_arg);
+  ~Guardian();
 
-  struct GUARD_NODE
-  {
-    Instance *instance;
-    /* state of an instance (i.e. STARTED, CRASHED, etc.) */
-    enum_instance_state state;
-    /* the amount of attemts to restart instance (cleaned up at success) */
-    int restart_counter;
-    /* triggered at a crash */
-    time_t crash_moment;
-    /* General time field. Used to provide timeouts (at shutdown and restart) */
-    time_t last_checked;
-  };
-
-  /* Return client state name. */
-  static const char *get_instance_state_name(enum_instance_state state);
+  void init();
 
-  Guardian(Thread_registry *thread_registry_arg,
-           Instance_map *instance_map_arg,
-           uint monitoring_interval_arg);
-  virtual ~Guardian();
-  /* Initialize or refresh the list of guarded instances */
-  int init();
-  /* Request guardian shutdown. Stop instances if needed */
+public:
   void request_shutdown();
-  /* Start instance protection */
-  int guard(Instance *instance, bool nolock= FALSE);
-  /* Stop instance protection */
-  int stop_guard(Instance *instance);
-  /* Returns TRUE if guardian thread is stopped */
-  int is_stopped();
+
+  bool is_stopped();
+
   void lock();
   void unlock();
 
-  /*
-    Return an internal list node for the given instance if the instance is
-    managed by Guardian. Otherwise, return NULL.
+  void ping();
 
-    MT-NOTE: must be called under acquired lock.
-  */
-  LIST *find_instance_node(Instance *instance);
+protected:
+  virtual void run();
+
+private:
+  void stop_instances();
 
-  /* The operation is used to check if the instance is active or not. */
-  bool is_active(Instance *instance);
+  void process_instance(Instance *instance);
 
+private:
   /*
-    Return state of the given instance list node. The pointer must specify
-    a valid list node.
+    LOCK_guardian protectes the members in this section:
+      - shutdown_requested;
+      - stopped;
+
+    Also, it is used for COND_guardian.
   */
-  inline enum_instance_state get_instance_state(LIST *instance_node);
-protected:
-  /* Main funtion of the thread */
-  virtual void run();
+  pthread_mutex_t LOCK_guardian;
 
-public:
+  /*
+    Guardian's main loop waits on this condition. So, it should be signalled
+    each time, when instance state has been changed and we want Guardian to
+    wake up.
+
+    TODO: Change this to having data-scoped conditions, i.e. conditions,
+    which indicate that some data has been changed.
+  */
   pthread_cond_t COND_guardian;
 
-private:
-  /* Prepares Guardian shutdown. Stops instances is needed */
-  int stop_instances();
-  /* check instance state and act accordingly */
-  void process_instance(Instance *instance, GUARD_NODE *current_node,
-                        LIST **guarded_instances, LIST *elem);
+  /*
+    This variable is set to TRUE, when Manager thread is shutting down.
+    The flag is used by Guardian thread to understand that it's time to
+    finish.
+  */
+  bool shutdown_requested;
+
+  /*
+    This flag is set to TRUE on shutdown by Guardian thread, when all guarded
+    mysqlds are stopped.
 
-  int stopped;
+    The flag is used in the Manager thread to wait for Guardian to stop all
+    mysqlds.
+  */
+  bool stopped;
 
-private:
-  pthread_mutex_t LOCK_guardian;
   Thread_info thread_info;
-  int monitoring_interval;
   Thread_registry *thread_registry;
   Instance_map *instance_map;
-  LIST *guarded_instances;
-  MEM_ROOT alloc;
-  /* this variable is set to TRUE when we want to stop Guardian thread */
-  bool shutdown_requested;
-};
-
 
-inline Guardian::enum_instance_state
-Guardian::get_instance_state(LIST *instance_node)
-{
-  return ((GUARD_NODE *) instance_node->data)->state;
-}
+private:
+  Guardian(const Guardian &);
+  Guardian&operator =(const Guardian &);
+};
 
 #endif /* INCLUDES_MYSQL_INSTANCE_MANAGER_GUARDIAN_H */