Partial fix for BUG#14106: IM: im_life_cycle and im_utils

tests fail on FreeBSD.

The patch contains of the following:
  - make Instance Manager, running in the daemon mode, dump
    the pid of angel-process in the special file;
  - default value of angel-pid-file-name is 'mysqlmanager.angel.pid';
  - if ordinary (IM) pid-file-name is specified in the configuration,
    angel-pid-file-name is updated according to the following
    rule: extension of the basename of pid-file-name is replaced by
    '.angel.pid.
    For example:
    - pid-file-name: /tmp/im.pid
      => angel-pid-file-name: /tmp/im.angel.pid
    - pid-file-name: /tmp/im.txt
      => angel-pid-file-name: /tmp/im.angel.pid
    - pid-file-name: /tmp/5.0/im
      => angel-pid-file-name: /tmp/5.0/im.angel.pid
  - add support for configuration option to customize angel
    pid file name;
  - fix test suite to use angel pid to kill Instance Manager
    by all means if something went wrong.

Background
----------

The problem is that on some OSes (FreeBSD for one) Instance
Manager does not get SIGTERM, so can not shutdown gracefully.
Test suite wasn't able to cope with it, so this leads to the
mess in test results.

The problem should be split into two:
  - fix signal handling;
  - fix test suite.

This patch fixes test suite so that it will be able to kill
uncooperative Instance Manager. In order to achieve this,
test suite needs to know PID of IM Angel process.


mysql-test/lib/mtr_process.pl:
  Added a function to send a signal to a process.
mysql-test/mysql-test-run.pl:
  Changed procedure of stopping Instance Manager.
  1. Try to stop IM normally (by sending SIGTERM);
  2. If one of IM-related processes is still alive,
  kill them all by SIGKILL and complain in the log.
server-tools/instance-manager/manager.cc:
  Made create_pid_file() available for the whole project.
server-tools/instance-manager/manager.h:
  Made create_pid_file() available for the whole project.
server-tools/instance-manager/mysqlmanager.cc:
  Dump PID of angel process into file.
server-tools/instance-manager/options.cc:
  Added an option to allow to customize angel pid file name.
server-tools/instance-manager/options.h:
  Added an option to allow to customize angel pid file name.

7 files changed, 169 insertions, 18 deletions

diff --git a/mysql-test/lib/mtr_process.pl b/mysql-test/lib/mtr_process.pl
index 973655a3eb6..6fa6bc73bdb 100644
--- a/mysql-test/lib/mtr_process.pl
+++ b/mysql-test/lib/mtr_process.pl
@@ -20,6 +20,7 @@ sub mtr_record_dead_children ();
 sub mtr_exit ($);
 sub sleep_until_file_created ($$$);
 sub mtr_kill_processes ($);
+sub mtr_kill_process ($$$$);
 
 # static in C
 sub spawn_impl ($$$$$$$$);
@@ -885,6 +886,25 @@ sub mtr_kill_processes ($) {
   }
 }
 
+
+sub mtr_kill_process ($$$$) {
+  my $pid= shift;
+  my $signal= shift;
+  my $retries= shift;
+  my $timeout= shift;
+
+  while (1)
+  {
+    kill($signal, $pid);
+
+    last unless kill (0, $pid) and $retries--;
+
+    mtr_debug("Sleep $timeout second waiting for processes to die");
+
+    sleep($timeout);
+  }
+}
+
 ##############################################################################
 #
 #  When we exit, we kill off all children
diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl
index 3a5fa2e7287..7613e7f087c 100755
--- a/mysql-test/mysql-test-run.pl
+++ b/mysql-test/mysql-test-run.pl
@@ -924,6 +924,7 @@ sub command_line_setup () {
    path_err =>        "$opt_vardir/log/im.err",
    path_log =>        "$opt_vardir/log/im.log",
    path_pid =>        "$opt_vardir/run/im.pid",
+   path_angel_pid =>  "$opt_vardir/run/im.angel.pid",
    path_sock =>       "$sockdir/im.sock",
    port =>            $im_port,
    start_timeout =>   $master->[0]->{'start_timeout'},
@@ -1179,6 +1180,7 @@ sub environment_setup () {
   $ENV{'NDB_STATUS_OK'}=      "YES";
 
   $ENV{'IM_PATH_PID'}=        $instance_manager->{path_pid};
+  $ENV{'IM_PATH_ANGEL_PID'}=  $instance_manager->{path_angel_pid};
   $ENV{'IM_PORT'}=            $instance_manager->{port};
 
   $ENV{'IM_MYSQLD1_SOCK'}=    $instance_manager->{instances}->[0]->{path_sock};
@@ -1803,6 +1805,7 @@ sub im_create_defaults_file($) {
 
 [manager]
 pid-file            = $instance_manager->{path_pid}
+angel-pid-file      = $instance_manager->{path_angel_pid}
 socket              = $instance_manager->{path_sock}
 port                = $instance_manager->{port}
 password-file       = $instance_manager->{password_file}
@@ -1827,7 +1830,7 @@ log-slow-queries    = $instance->{path_datadir}/mysqld$server_id.slow.log
 language            = $path_language
 character-sets-dir  = $path_charsetsdir
 basedir             = $path_my_basedir
-server_id           =$server_id
+server_id           = $server_id
 skip-stack-trace
 skip-innodb
 skip-bdb
@@ -2786,6 +2789,18 @@ sub im_start($$) {
 sub im_stop($) {
   my $instance_manager = shift;
 
+  # Obtain mysqld-process pids before we start stopping IM (it can delete pid
+  # files).
+
+  my @mysqld_pids = ();
+  my $instances = $instance_manager->{'instances'};
+
+  push(@mysqld_pids, mtr_get_pid_from_file($instances->[0]->{'path_pid'}))
+    if -r $instances->[0]->{'path_pid'};
+
+  push(@mysqld_pids, mtr_get_pid_from_file($instances->[1]->{'path_pid'}))
+    if -r $instances->[1]->{'path_pid'};
+
   # Re-read pid from the file, since during tests Instance Manager could have
   # been restarted, so its pid could have been changed.
 
@@ -2793,34 +2808,79 @@ sub im_stop($) {
     mtr_get_pid_from_file($instance_manager->{'path_pid'})
       if -f $instance_manager->{'path_pid'};
 
+  if (-f $instance_manager->{'path_angel_pid'})
+  {
+    $instance_manager->{'angel_pid'} =
+      mtr_get_pid_from_file($instance_manager->{'path_angel_pid'})
+  }
+  else
+  {
+    $instance_manager->{'angel_pid'} = undef;
+  }
+
   # Inspired from mtr_stop_mysqld_servers().
 
   start_reap_all();
 
-  # Create list of pids. We should stop Instance Manager and all started
-  # mysqld-instances. Some of them may be nonguarded, so IM will not stop them
-  # on shutdown.
+  # Try graceful shutdown.
 
-  my @pids = ( $instance_manager->{'pid'} );
-  my $instances = $instance_manager->{'instances'};
+  mtr_kill_process($instance_manager->{'pid'}, 'TERM', 10, 1);
+
+  # Check that all processes died.
+
+  my $clean_shutdown= 0;
 
-  if ( -r $instances->[0]->{'path_pid'} )
+  while (1)
   {
-    push(@pids, mtr_get_pid_from_file($instances->[0]->{'path_pid'}));
+    last if kill (0, $instance_manager->{'pid'});
+
+    last if (defined $instance_manager->{'angel_pid'}) &&
+            kill (0, $instance_manager->{'angel_pid'});
+
+    foreach my $pid (@mysqld_pids)
+    {
+      last if kill (0, $pid);
+    }
+
+    $clean_shutdown= 1;
+    last;
   }
 
-  if ( -r $instances->[1]->{'path_pid'} )
+  # Kill leftovers (the order is important).
+
+  unless ($clean_shutdown)
   {
-    push(@pids, mtr_get_pid_from_file($instances->[1]->{'path_pid'}));
-  }
+    mtr_kill_process($instance_manager->{'angel_pid'}, 'KILL', 10, 1)
+      if defined $instance_manager->{'angel_pid'};
+    
+    mtr_kill_process($instance_manager->{'pid'}, 'KILL', 10, 1);
 
-  # Kill processes.
+    # Shutdown managed mysqld-processes. Some of them may be nonguarded, so IM
+    # will not stop them on shutdown. So, we should firstly try to end them
+    # legally.
+
+    mtr_kill_processes(\@mysqld_pids);
+
+    # Complain in error log so that a warning will be shown.
+
+    my $errlog= "$opt_vardir/log/mysql-test-run.pl.err";
+
+    open (ERRLOG, ">>$errlog") ||
+      mtr_error("Can not open error log ($errlog)");
+
+    my $ts= localtime();
+    print ERRLOG
+      "Warning: [$ts] Instance Manager did not shutdown gracefully.\n";
+
+    close ERRLOG;
+  }
 
-  mtr_kill_processes(\@pids);
+  # That's all.
 
   stop_reap_all();
 
   $instance_manager->{'pid'} = undef;
+  $instance_manager->{'angel_pid'} = undef;
 }
 
 
diff --git a/server-tools/instance-manager/manager.cc b/server-tools/instance-manager/manager.cc
index 95f9029f648..90d9d04cd36 100644
--- a/server-tools/instance-manager/manager.cc
+++ b/server-tools/instance-manager/manager.cc
@@ -35,12 +35,12 @@
 #endif
 
 
-static int create_pid_file(const char *pid_file_name)
+int create_pid_file(const char *pid_file_name, int pid)
 {
   if (FILE *pid_file= my_fopen(pid_file_name,
                                O_WRONLY | O_CREAT | O_BINARY, MYF(0)))
   {
-    fprintf(pid_file, "%d\n", (int) getpid());
+    fprintf(pid_file, "%d\n", (int) pid);
     my_fclose(pid_file, MYF(0));
     return 0;
   }
@@ -138,8 +138,13 @@ void manager(const Options &options)
   if (user_map.load(options.password_file_name))
     return;
 
-  /* write pid file */
-  if (create_pid_file(options.pid_file_name))
+  /* write Instance Manager pid file */
+
+  log_info("IM pid file: '%s'; PID: %d.",
+           (const char *) options.pid_file_name,
+           (int) manager_pid);
+
+  if (create_pid_file(options.pid_file_name, manager_pid))
     return;
 
   sigset_t mask;
diff --git a/server-tools/instance-manager/manager.h b/server-tools/instance-manager/manager.h
index 12ed6b3b1ff..3ddf292132e 100644
--- a/server-tools/instance-manager/manager.h
+++ b/server-tools/instance-manager/manager.h
@@ -20,4 +20,6 @@ struct Options;
 
 void manager(const Options &options);
 
+int create_pid_file(const char *pid_file_name, int pid);
+
 #endif // INCLUDES_MYSQL_INSTANCE_MANAGER_MANAGER_H
diff --git a/server-tools/instance-manager/mysqlmanager.cc b/server-tools/instance-manager/mysqlmanager.cc
index d0b2cf2666c..ef714099de7 100644
--- a/server-tools/instance-manager/mysqlmanager.cc
+++ b/server-tools/instance-manager/mysqlmanager.cc
@@ -338,6 +338,14 @@ spawn:
     /* Here we return to main, and fall into manager */
     break;
   default:                                    // parent, success
+    pid= getpid(); /* Get our pid. */
+
+    log_info("Angel pid file: '%s'; PID: %d.",
+             (const char *) options.angel_pid_file_name,
+             (int) pid);
+
+    create_pid_file(Options::angel_pid_file_name, pid);
+
     while (child_status == CHILD_OK && is_terminated == 0)
       sigsuspend(&zeromask);
 
diff --git a/server-tools/instance-manager/options.cc b/server-tools/instance-manager/options.cc
index e7d366e7457..4d71680e061 100644
--- a/server-tools/instance-manager/options.cc
+++ b/server-tools/instance-manager/options.cc
@@ -44,6 +44,7 @@ const char *Options::user= 0;                   /* No default value */
 const char *default_password_file_name= QUOTE(DEFAULT_PASSWORD_FILE_NAME);
 const char *default_log_file_name= QUOTE(DEFAULT_LOG_FILE_NAME);
 const char *Options::config_file= QUOTE(DEFAULT_CONFIG_FILE);
+const char *Options::angel_pid_file_name= NULL;
 #endif
 const char *Options::log_file_name= default_log_file_name;
 const char *Options::pid_file_name= QUOTE(DEFAULT_PID_FILE_NAME);
@@ -58,6 +59,9 @@ char **Options::saved_argv= NULL;
 /* Remember if the config file was forced */
 bool Options::is_forced_default_file= 0;
 
+static const char * const ANGEL_PID_FILE_SUFFIX= ".angel.pid";
+static const int ANGEL_PID_FILE_SUFFIX_LEN= strlen(ANGEL_PID_FILE_SUFFIX);
+
 /*
   List of options, accepted by the instance manager.
   List must be closed with empty option.
@@ -72,6 +76,7 @@ enum options {
 #ifndef __WIN__
   OPT_RUN_AS_SERVICE,
   OPT_USER,
+  OPT_ANGEL_PID_FILE,
 #else
   OPT_INSTALL_SERVICE,
   OPT_REMOVE_SERVICE,
@@ -94,7 +99,14 @@ static struct my_option my_long_options[] =
 
   { "pid-file", OPT_PID_FILE, "Pid file to use.",
     (gptr *) &Options::pid_file_name, (gptr *) &Options::pid_file_name,
-   0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
+    0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
+
+#ifndef __WIN__
+  { "angel-pid-file", OPT_ANGEL_PID_FILE, "Pid file for angel process.",
+    (gptr *) &Options::angel_pid_file_name,
+    (gptr *) &Options::angel_pid_file_name,
+    0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
+#endif
 
   { "socket", OPT_SOCKET, "Socket file to use for connection.",
     (gptr *) &Options::socket_file_name, (gptr *) &Options::socket_file_name,
@@ -290,6 +302,46 @@ int Options::load(int argc, char **argv)
                       get_one_option)) != 0)
     goto err;
 
+#ifndef __WIN__
+  if (Options::run_as_service)
+  {
+    if (Options::angel_pid_file_name == NULL)
+    {
+      /*
+        Calculate angel pid file on the IM pid file basis: replace the
+        extension (everything after the last dot) of the pid file basename to
+        '.angel.pid'.
+      */
+
+      char *angel_pid_file_name;
+      char *base_name_ptr;
+      char *ext_ptr;
+
+      angel_pid_file_name= (char *) malloc(strlen(Options::pid_file_name) +
+                                           ANGEL_PID_FILE_SUFFIX_LEN);
+
+      strcpy(angel_pid_file_name, Options::pid_file_name);
+
+      base_name_ptr= strrchr(angel_pid_file_name, '/');
+
+      if (!base_name_ptr)
+        base_name_ptr= angel_pid_file_name + 1;
+
+      ext_ptr= strrchr(base_name_ptr, '.');
+      if (ext_ptr)
+        *ext_ptr= 0;
+
+      strcat(angel_pid_file_name, ANGEL_PID_FILE_SUFFIX);
+
+      Options::angel_pid_file_name= angel_pid_file_name;
+    }
+    else
+    {
+      Options::angel_pid_file_name= strdup(Options::angel_pid_file_name);
+    }
+  }
+#endif
+
   return 0;
 
 err:
@@ -301,6 +353,9 @@ void Options::cleanup()
   /* free_defaults returns nothing */
   if (Options::saved_argv != NULL)
     free_defaults(Options::saved_argv);
+
+  if (Options::run_as_service)
+    free((void *) Options::angel_pid_file_name);
 }
 
 #ifdef __WIN__
diff --git a/server-tools/instance-manager/options.h b/server-tools/instance-manager/options.h
index abb094eac93..ad3458869b6 100644
--- a/server-tools/instance-manager/options.h
+++ b/server-tools/instance-manager/options.h
@@ -35,6 +35,7 @@ struct Options
 #else
   static char run_as_service;        /* handle_options doesn't support bool */
   static const char *user;
+  static const char *angel_pid_file_name;
 #endif
   static bool is_forced_default_file;
   static const char *log_file_name;