Fix for BUG#24415: Instance manager test im_daemon_life_cycle fails randomly.

The cause of im_daemon_life_cycle.imtest random failures was the following
behaviour of some implementations of LINUX threads: let's suppose that a
process has several threads (in LINUX threads, there is a separate process for
each thread). When the main process gets killed, the parent receives SIGCHLD
before all threads (child processes) die. In other words, the parent receives
SIGCHLD, when its child is not completely dead.

In terms of IM, that means that IM-angel receives SIGCHLD when IM-main is not dead
and still holds some resources. After receiving SIGCHLD, IM-angel restarts
IM-main, but IM-main failed to initialize, because previous instance (copy) of
IM-main still holds server socket (TCP-port).

Another problem here was that IM-angel restarted IM-main only if it was killed
by signal. If it exited with error, IM-angel thought it's intended / graceful
shutdown and exited itself.

So, when the second instance of IM-main failed to initialize, IM-angel thought
it's intended shutdown and quit.

The fix is
  1. to change IM-angel so that it restarts IM-main if it exited with error code;
  2. to change IM-main so that it returns proper exit code in case of failure.


mysql-test/t/disabled.def:
  Enable im_daemon_life_cycle.
server-tools/instance-manager/listener.cc:
  Set error status if Listener failed to initialize.
server-tools/instance-manager/manager.cc:
  Return exit code from the manager.
server-tools/instance-manager/manager.h:
  Return exit code from the manager.
server-tools/instance-manager/mysqlmanager.cc:
  1. Restart IM-main if exit code is not EXIT_SUCCESS (0).
  2. Log IM-main exit code in case of failure.
server-tools/instance-manager/thread_registry.cc:
  Add support for exit code.
server-tools/instance-manager/thread_registry.h:
  Add support for exit code.

6 files changed, 51 insertions, 21 deletions

diff --git a/server-tools/instance-manager/listener.cc b/server-tools/instance-manager/listener.cc
index 15583b233fb..a055d98656c 100644
--- a/server-tools/instance-manager/listener.cc
+++ b/server-tools/instance-manager/listener.cc
@@ -210,10 +210,13 @@ void Listener_thread::run()
   return;
 
 err:
+  log_error("Listener: failed to initialize. Initiate shutdown...");
+
   // we have to close the ip sockets in case of error
   for (i= 0; i < num_sockets; i++)
     closesocket(sockets[i]);
 
+  thread_registry.set_error_status();
   thread_registry.unregister_thread(&thread_info);
   thread_registry.request_shutdown();
   my_thread_end();
diff --git a/server-tools/instance-manager/manager.cc b/server-tools/instance-manager/manager.cc
index 30b6aea8519..cadb7a4aa0c 100644
--- a/server-tools/instance-manager/manager.cc
+++ b/server-tools/instance-manager/manager.cc
@@ -114,6 +114,9 @@ void stop_all(Guardian_thread *guardian, Thread_registry *registry)
   pthread_cond_signal(&guardian->COND_guardian);
   /* stop all threads */
   registry->deliver_shutdown();
+
+  /* Set error status in the thread registry. */
+  registry->set_error_status();
 }
 
 /*
@@ -123,7 +126,7 @@ void stop_all(Guardian_thread *guardian, Thread_registry *registry)
   architecture.
 */
 
-void manager(const Options &options)
+int manager(const Options &options)
 {
   Thread_registry thread_registry;
   /*
@@ -145,10 +148,10 @@ void manager(const Options &options)
   instance_map.guardian= &guardian_thread;
 
   if (instance_map.init() || user_map.init())
-    return;
+    return 1;
 
   if (user_map.load(options.password_file_name))
-    return;
+    return 1;
 
   /* write Instance Manager pid file */
 
@@ -157,7 +160,7 @@ void manager(const Options &options)
            (int) manager_pid);
 
   if (create_pid_file(options.pid_file_name, manager_pid))
-    return;
+    return 1;
 
   /*
     Initialize signals and alarm-infrastructure.
@@ -301,5 +304,6 @@ err:
   end_thr_alarm(1);
   /* don't pthread_exit to kill all threads who did not shut down in time */
 #endif
-}
 
+  return thread_registry.get_error_status() ? 1 : 0;
+}
diff --git a/server-tools/instance-manager/manager.h b/server-tools/instance-manager/manager.h
index 0d459d9cee1..22af0d39115 100644
--- a/server-tools/instance-manager/manager.h
+++ b/server-tools/instance-manager/manager.h
@@ -18,7 +18,7 @@
 
 struct Options;
 
-void manager(const Options &options);
+int manager(const Options &options);
 
 int create_pid_file(const char *pid_file_name, int pid);
 
diff --git a/server-tools/instance-manager/mysqlmanager.cc b/server-tools/instance-manager/mysqlmanager.cc
index 2b6921d59aa..83918092454 100644
--- a/server-tools/instance-manager/mysqlmanager.cc
+++ b/server-tools/instance-manager/mysqlmanager.cc
@@ -110,8 +110,7 @@ int main(int argc, char *argv[])
   else
 #endif
 
-  manager(options);
-  return_value= 0;
+  return_value= manager(options);
 
 err:
   options.cleanup();
@@ -254,26 +253,23 @@ static void daemonize(const char *log_file_name)
 enum { CHILD_OK= 0, CHILD_NEED_RESPAWN, CHILD_EXIT_ANGEL };
 
 static volatile sig_atomic_t child_status= CHILD_OK;
+static volatile sig_atomic_t child_exit_code= 0;
 
 /*
-  Signal handler for SIGCHLD: reap child, analyze child exit status, and set
+  Signal handler for SIGCHLD: reap child, analyze child exit code, and set
   child_status appropriately.
 */
 
 void reap_child(int __attribute__((unused)) signo)
 {
-  int child_exit_status;
-  /* As we have only one child, no need to cycle waitpid */
-  if (waitpid(0, &child_exit_status, WNOHANG) > 0)
+  /* NOTE: As we have only one child, no need to cycle waitpid(). */
+
+  int exit_code;
+
+  if (waitpid(0, &exit_code, WNOHANG) > 0)
   {
-    if (WIFSIGNALED(child_exit_status))
-      child_status= CHILD_NEED_RESPAWN;
-    else
-      /*
-        As reap_child is not called for SIGSTOP, we should be here only
-        if the child exited normally.
-      */
-      child_status= CHILD_EXIT_ANGEL;
+    child_exit_code= exit_code;
+    child_status= exit_code ? CHILD_NEED_RESPAWN : CHILD_EXIT_ANGEL;
   }
 }
 
@@ -353,7 +349,9 @@ spawn:
     else if (child_status == CHILD_NEED_RESPAWN)
     {
       child_status= CHILD_OK;
-      log_error("angel(): mysqlmanager exited abnormally: respawning...");
+      log_error("angel(): mysqlmanager exited abnormally (exit code: %d):"
+                "respawning...",
+                (int) child_exit_code);
       sleep(1); /* don't respawn too fast */
       goto spawn;
     }
diff --git a/server-tools/instance-manager/thread_registry.cc b/server-tools/instance-manager/thread_registry.cc
index e061bb3384b..5bb4206982e 100644
--- a/server-tools/instance-manager/thread_registry.cc
+++ b/server-tools/instance-manager/thread_registry.cc
@@ -53,6 +53,7 @@ Thread_info::Thread_info(pthread_t thread_id_arg) :
 Thread_registry::Thread_registry() :
    shutdown_in_progress(false)
   ,sigwait_thread_pid(pthread_self())
+  ,error_status(FALSE)
 {
   pthread_mutex_init(&LOCK_thread_registry, 0);
   pthread_cond_init(&COND_thread_registry_is_empty, 0);
@@ -243,3 +244,23 @@ void Thread_registry::request_shutdown()
 {
   pthread_kill(sigwait_thread_pid, SIGTERM);
 }
+
+
+int Thread_registry::get_error_status()
+{
+  int ret_error_status;
+
+  pthread_mutex_lock(&LOCK_thread_registry);
+  ret_error_status= error_status;
+  pthread_mutex_unlock(&LOCK_thread_registry);
+
+  return ret_error_status;
+}
+
+
+void Thread_registry::set_error_status()
+{
+  pthread_mutex_lock(&LOCK_thread_registry);
+  error_status= TRUE;
+  pthread_mutex_unlock(&LOCK_thread_registry);
+}
diff --git a/server-tools/instance-manager/thread_registry.h b/server-tools/instance-manager/thread_registry.h
index 694fa57378e..f6de49cd9ad 100644
--- a/server-tools/instance-manager/thread_registry.h
+++ b/server-tools/instance-manager/thread_registry.h
@@ -97,12 +97,16 @@ public:
                  pthread_mutex_t *mutex);
   int cond_timedwait(Thread_info *info, pthread_cond_t *cond,
                      pthread_mutex_t *mutex, struct timespec *wait_time);
+  int get_error_status();
+  void set_error_status();
+
 private:
   Thread_info head;
   bool shutdown_in_progress;
   pthread_mutex_t LOCK_thread_registry;
   pthread_cond_t COND_thread_registry_is_empty;
   pthread_t sigwait_thread_pid;
+  bool error_status;
 };